const processFile = (file) => { // do something with file file.close() // close file to prevent memory leaks }
queue.process(async (job) => { const file = await getFile(job.data.fileId) processFile(file) })
const heapdump = require('heapdump')
const processFile = (file) => { // do something with file heapdump.writeSnapshot() // write heapdump snapshot to locate memory leaks file.close() // close file to prevent memory leaks }
queue.process(async (job) => { const file = await getFile(job.data.fileId) processFile(file) })
const processFilesInChunks = async (files) => { const chunkSize = 100 // process files in chunks of 100 const numChunks = Math.ceil(files.length / chunkSize)
for (let i = 0; i < numChunks; i++) { const chunk = files.slice(i * chunkSize, (i + 1) * chunkSize)
await Promise.all(chunk.map(async (file) => {
// do something with file
await processFile(file)
}))
} }
const files = await generateFiles(2000) processFilesInChunks(files)