How to Recursively Fetch Files in Node.js
Greetings, friends! In this tutorial, I'll teach you how to get all the files in a directory recursively using Node.js. We'll create a simple folder structure for the purpose of proving that our code works.
If you want to create a directory structure just like this, then you can run the following script in your terminal:
mkdir javascript && cd $_ && touch index.js \
&& mkdir directory_1 directory_2 \
&& cd directory_1 && mkdir subdirectory_1 subdirectory_2 && touch dir-1.js \
&& cd subdirectory_1 && touch sub-1a.js sub-1b.js \
&& cd ../subdirectory_2 && touch sub-2a.js \
&& cd ../../directory_2 && touch dir-2.js \
&& cd ../..
I'll make use of the fs Promises API along with async/await syntax to make the code cleaner and easier to understand. For those who don't know, Node provides an alternative set of asynchronous file system methods that return Promises instead instead of using callbacks. For example, you would often use fs.readFile to read the contents of a file asynchronously using an error-first callback:
fs.readFile('/etc/passwd', (err, data) => {
if (err)
throw err
console.log(data.toString())
})
Using the fs Promises API instead:
async function run() {
try {
const data = await fs.promises.readFile('/etc/passwd')
console.log(data.toString())
}
catch (err) {
throw err
}
}
run()
I'll also use fs.Stats objects which provide useful utility methods for determining whether an object is a file or directory.
The code below lets you recursively fetch files. We will pick a directory named javascript as the root directory and create an fs.Stats object for every file or directory. If the object is a file, then we will append it to the fetchedFiles array. If the object is a directory, then we will push to our files array. Notice that we are already looping through this array. JavaScript lets us continue adding to this array as we iterate through it to keep the loop going. Effectively, this means that we will continue looping through the files array and searching through each directory recursively until we encounter no more files or other directories. The path.join method will help us create a file path.
const fs = require('node:fs')
const path = require('node:path')
async function fetchFiles(targetPath) {
const files = await fs.promises.readdir(targetPath)
const fetchedFiles = []
for (const file of files) {
try {
const filepath = path.join(targetPath, file)
const stats = await fs.promises.lstat(filepath)
if (stats.isFile())
fetchedFiles.push({ filepath })
if (stats.isDirectory()) {
const childFiles = await fs.promises.readdir(filepath)
files.push(...childFiles.map(f => path.join(file, f)))
}
}
catch (err) {
console.error(err)
}
}
return fetchedFiles
}
async function run() {
try {
const files = await fetchFiles('javascript')
console.log(files)
}
catch (err) {
console.error(err)
}
}
run()
The output should look like the following:
[{ filepath: 'javascript/index.js' }, { filepath: 'javascript/directory_1/dir-1.js' }, { filepath: 'javascript/directory_2/dir-2.js' }, { filepath: 'javascript/directory_1/subdirectory_1/sub-1a.js' }, { filepath: 'javascript/directory_1/subdirectory_1/sub-1b.js' }, { filepath: 'javascript/directory_1/subdirectory_2/sub-2a.js' }]
If you happen to have multiple file types in one or more directories and you want to only fetch JavaScript files, you could replace
if (stats.isFile())
fetchedFiles.push({ filepath })
With the following:
if (stats.isFile() && file.slice(-3) === '.js')
fetchedFiles.push({ filepath })
Conclusion
There are multiple ways to recursively fetch files in Node.js. I find this technique quick and easy to implement. I hope it helps!