From eb9bd85efdb3ca28acca18a39a9dd1cd964815e3 Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Mon, 6 Oct 2025 19:11:54 +0200 Subject: [PATCH 1/7] feat: Blog post on reading and writing files --- .../reading-writing-files-nodejs/index.md | 717 ++++++++++++++++++ 1 file changed, 717 insertions(+) create mode 100644 src/content/blog/reading-writing-files-nodejs/index.md diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md new file mode 100644 index 0000000..ea7332b --- /dev/null +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -0,0 +1,717 @@ +--- +date: 2025-01-06T10:00:00 +updatedAt: 2025-01-06T10:00:00 +title: Reading and Writing Files in Node.js - The Complete Modern Guide +slug: reading-writing-files-nodejs +description: Learn the modern way to read and write files in Node.js using promises, streams, and file handles. Master memory-efficient file operations for production applications. +authors: ['luciano-mammino'] +tags: ['blog'] +--- + +File operations are at the heart of most Node.js applications. Whether you're building a web server that serves static assets, processing CSV files, handling user uploads, or working with configuration files, knowing how to efficiently read and write files is absolutely essential. + +In this comprehensive guide, we'll explore the modern approaches to file handling in Node.js. We'll start with the simplest methods using promises, then dive into more advanced techniques like streaming and low-level file operations that can handle massive files without breaking your application. + +Let's dive in and master file operations in Node.js! + +## Why File Operations Matter + +Before we jump into the code, let's talk about why file operations are so crucial in Node.js development. Almost every application you'll build will need to: + +- **Read configuration files** (JSON, YAML, environment files) +- **Process user uploads** (images, documents, data files) +- **Generate reports** (CSV exports, PDFs, logs) +- **Cache data** (temporary files, session storage) +- **Serve static content** (HTML, CSS, JavaScript files) + +The way you handle these operations can make or break your application's performance, especially when dealing with large files or high traffic. + +## Reading and Writing Files with Promises + +The most straightforward way to work with files in modern Node.js is using the `fs/promises` module. This gives us a clean, promise-based API that works beautifully with `async/await`. + +### Reading Files with fs/promises + +Here's how you can read a file using the modern promise-based approach: + +```javascript +import { readFile } from 'fs/promises' + +async function readConfigFile() { + try { + const data = await readFile('config.json', 'utf8') + const config = JSON.parse(data) + console.log('Configuration loaded:', config) + return config + } catch (error) { + console.error('Failed to read config file:', error.message) + throw error + } +} + +// Usage +const config = await readConfigFile() +``` + +The `readFile` function loads the entire file content into memory and returns it as a string (when you specify an encoding like 'utf8') or as a Buffer (when no encoding is specified). + +### Writing Files with fs/promises + +Writing files is just as straightforward: + +```javascript +import { writeFile } from 'fs/promises' + +async function saveUserData(userData) { + try { + const jsonData = JSON.stringify(userData, null, 2) + await writeFile('user-data.json', jsonData, 'utf8') + console.log('User data saved successfully!') + } catch (error) { + console.error('Failed to save user data:', error.message) + throw error + } +} + +// Usage +await saveUserData({ name: 'John Doe', email: 'john@example.com' }) +``` + +### Reading and Writing Binary Files + +Not all files are text-based. Here's how you handle binary files like images: + +```javascript +import { readFile, writeFile } from 'fs/promises' + +async function copyImage(sourcePath, destinationPath) { + try { + // Read binary data (no encoding specified) + const imageData = await readFile(sourcePath) + + // Write binary data + await writeFile(destinationPath, imageData) + + console.log('Image copied successfully!') + console.log(`File size: ${imageData.length} bytes`) + } catch (error) { + console.error('Failed to copy image:', error.message) + throw error + } +} + +// Usage +await copyImage('original.jpg', 'copy.jpg') +``` + +### Working with Directories + +You'll often need to work with directories too: + +```javascript +import { readdir, mkdir, stat } from 'fs/promises' +import { join } from 'path' + +async function processDirectory(dirPath) { + try { + // Create directory if it doesn't exist + await mkdir('processed', { recursive: true }) + + // Read directory contents + const files = await readdir(dirPath) + + for (const file of files) { + const filePath = join(dirPath, file) + const stats = await stat(filePath) + + if (stats.isFile()) { + console.log(`Processing file: ${file} (${stats.size} bytes)`) + // Process file here... + } + } + } catch (error) { + console.error('Directory processing failed:', error.message) + throw error + } +} + +// Usage +await processDirectory('./uploads') +``` + +## The Memory Problem: Why Promises Aren't Always Enough + +The promise-based approach we've seen so far is perfect for small to medium-sized files. However, there's a significant limitation: **everything gets loaded into memory at once**. + +Imagine you're trying to read a 2GB log file using `readFile()`. Your Node.js process will attempt to load all 2GB into memory simultaneously. This can lead to several problems: + +1. **Out of memory errors** - Your application might crash +2. **Poor performance** - High memory usage affects other operations +3. **Blocking behavior** - Large file operations can block your event loop + +### Understanding Node.js Buffer Limits + +Node.js has built-in limits on buffer sizes to prevent applications from consuming too much memory. You can check these limits: + +```javascript +// Check the maximum buffer size +console.log('Max buffer size:', Buffer.constants.MAX_LENGTH) +console.log('Max string length:', Buffer.constants.MAX_STRING_LENGTH) + +// On most systems: +// MAX_LENGTH is around 2GB (2,147,483,647 bytes on 64-bit systems) +// MAX_STRING_LENGTH is around 1GB +``` + +If you try to read a file larger than these limits using `readFile()`, you'll get an error: + +```javascript +import { readFile } from 'fs/promises' + +try { + // This will fail if the file is larger than the buffer limit + const hugeFile = await readFile('massive-dataset.csv', 'utf8') +} catch (error) { + if (error.code === 'ERR_FS_FILE_TOO_LARGE') { + console.log('File is too large to read into memory at once!') + } +} +``` + +### When to Use Promise-Based Methods + +Promise-based file operations are great when: + +- **Files are small to medium-sized** (typically under 100MB) +- **You need the entire content at once** (parsing JSON, reading config files) +- **Simplicity is important** (rapid prototyping, simple scripts) +- **Memory usage isn't a concern** (plenty of RAM available) + +## Low-Level File Operations with File Handles + +When you need more control over file operations, Node.js provides lower-level APIs using file handles. This approach allows you to read and write files incrementally, giving you fine-grained control over memory usage. + +### Opening and Working with File Handles + +```javascript +import { open } from 'fs/promises' + +async function readFileInChunks(filePath) { + let fileHandle + + try { + // Open the file for reading + fileHandle = await open(filePath, 'r') + + const chunkSize = 1024 // Read 1KB at a time + const buffer = Buffer.alloc(chunkSize) + let position = 0 + let totalBytesRead = 0 + + while (true) { + // Read a chunk from the current position + const result = await fileHandle.read(buffer, 0, chunkSize, position) + + if (result.bytesRead === 0) { + // End of file reached + break + } + + // Process the chunk + const chunk = buffer.subarray(0, result.bytesRead) + console.log(`Read ${result.bytesRead} bytes:`, chunk.toString('utf8')) + + position += result.bytesRead + totalBytesRead += result.bytesRead + } + + console.log(`Total bytes read: ${totalBytesRead}`) + + } catch (error) { + console.error('Error reading file:', error.message) + throw error + } finally { + // Always close the file handle + if (fileHandle) { + await fileHandle.close() + } + } +} + +// Usage +await readFileInChunks('large-file.txt') +``` + +### Writing Files Incrementally + +```javascript +import { open } from 'fs/promises' + +async function writeFileInChunks(filePath, data) { + let fileHandle + + try { + // Open file for writing (creates if doesn't exist) + fileHandle = await open(filePath, 'w') + + const chunkSize = 1024 + const buffer = Buffer.from(data, 'utf8') + let position = 0 + + while (position < buffer.length) { + const remainingBytes = buffer.length - position + const bytesToWrite = Math.min(chunkSize, remainingBytes) + + const chunk = buffer.subarray(position, position + bytesToWrite) + const result = await fileHandle.write(chunk, 0, bytesToWrite, position) + + console.log(`Wrote ${result.bytesWritten} bytes at position ${position}`) + position += result.bytesWritten + } + + console.log('File writing completed!') + + } catch (error) { + console.error('Error writing file:', error.message) + throw error + } finally { + if (fileHandle) { + await fileHandle.close() + } + } +} + +// Usage +const largeText = 'A'.repeat(10000) // 10KB of 'A' characters +await writeFileInChunks('output.txt', largeText) +``` + +### Why File Handles Are Low-Level + +While file handles give you precise control, they require you to manage a lot of details manually: + +- **Buffer management** - You need to allocate and manage buffers +- **Position tracking** - You must keep track of read/write positions +- **Error handling** - More complex error scenarios to handle +- **Resource cleanup** - You must remember to close file handles + +This low-level approach is powerful but verbose. For most use cases, there's a better solution: **streams**. + +## Streams: The Best of Both Worlds + +Streams provide the memory efficiency of low-level file operations with a much more ergonomic API. They're perfect for processing large files without loading everything into memory. + +### Understanding Stream Advantages + +Streams offer several key benefits: + +1. **Memory efficiency** - Process data piece by piece +2. **Composability** - Chain multiple operations together +3. **Backpressure handling** - Automatically manage fast producers and slow consumers +4. **Event-driven** - React to data as it becomes available + +### Reading Files with Streams + +Here's how to read a file using a readable stream: + +```javascript +import { createReadStream } from 'fs' +import { pipeline } from 'stream/promises' + +async function processLargeFile(filePath) { + const readStream = createReadStream(filePath, { + encoding: 'utf8', + highWaterMark: 16 * 1024 // 16KB chunks + }) + + let lineCount = 0 + let charCount = 0 + + readStream.on('data', (chunk) => { + // Process chunk as it arrives + charCount += chunk.length + lineCount += (chunk.match(/\n/g) || []).length + + console.log(`Processed chunk: ${chunk.length} characters`) + }) + + readStream.on('end', () => { + console.log(`File processing complete!`) + console.log(`Total characters: ${charCount}`) + console.log(`Total lines: ${lineCount}`) + }) + + readStream.on('error', (error) => { + console.error('Stream error:', error.message) + }) +} + +// Usage +await processLargeFile('huge-log-file.txt') +``` + +### Writing Files with Streams + +Writing with streams is equally straightforward: + +```javascript +import { createWriteStream } from 'fs' +import { pipeline } from 'stream/promises' +import { Readable } from 'stream' + +async function generateLargeFile(filePath, numberOfLines) { + const writeStream = createWriteStream(filePath, { + encoding: 'utf8', + highWaterMark: 64 * 1024 // 64KB buffer + }) + + // Create a readable stream that generates data + const dataGenerator = new Readable({ + read() { + if (numberOfLines > 0) { + this.push(`Line ${numberOfLines}: This is some sample data\n`) + numberOfLines-- + } else { + this.push(null) // End the stream + } + } + }) + + try { + // Use pipeline for proper error handling and cleanup + await pipeline(dataGenerator, writeStream) + console.log('File generation completed!') + } catch (error) { + console.error('Pipeline failed:', error.message) + throw error + } +} + +// Usage: Generate a file with 1 million lines +await generateLargeFile('generated-data.txt', 1000000) +``` + +### Stream Composition and Processing + +One of the most powerful features of streams is the ability to compose them. Here's an example that reads a CSV file, processes it, and writes the results: + +```javascript +import { createReadStream, createWriteStream } from 'fs' +import { pipeline } from 'stream/promises' +import { Transform } from 'stream' + +// Custom transform stream to process CSV data +class CSVProcessor extends Transform { + constructor(options = {}) { + super({ objectMode: true, ...options }) + this.headers = null + this.lineCount = 0 + } + + _transform(chunk, encoding, callback) { + const lines = chunk.toString().split('\n') + + for (const line of lines) { + if (!line.trim()) continue + + if (!this.headers) { + this.headers = line.split(',') + continue + } + + const values = line.split(',') + const record = {} + + this.headers.forEach((header, index) => { + record[header.trim()] = values[index]?.trim() || '' + }) + + // Transform the record (e.g., uppercase all names) + if (record.name) { + record.name = record.name.toUpperCase() + } + + this.push(JSON.stringify(record) + '\n') + this.lineCount++ + } + + callback() + } + + _flush(callback) { + console.log(`Processed ${this.lineCount} records`) + callback() + } +} + +async function processCSVFile(inputPath, outputPath) { + try { + await pipeline( + createReadStream(inputPath, { encoding: 'utf8' }), + new CSVProcessor(), + createWriteStream(outputPath, { encoding: 'utf8' }) + ) + + console.log('CSV processing completed!') + } catch (error) { + console.error('CSV processing failed:', error.message) + throw error + } +} + +// Usage +await processCSVFile('users.csv', 'processed-users.json') +``` + +### Handling Backpressure + +Streams automatically handle backpressure - the situation where data is being produced faster than it can be consumed. Here's how you can monitor and control it: + +```javascript +import { createReadStream, createWriteStream } from 'fs' + +function copyFileWithBackpressureHandling(source, destination) { + return new Promise((resolve, reject) => { + const readStream = createReadStream(source) + const writeStream = createWriteStream(destination) + + readStream.on('data', (chunk) => { + const canContinue = writeStream.write(chunk) + + if (!canContinue) { + // Backpressure detected - pause reading + console.log('Backpressure detected, pausing read stream') + readStream.pause() + + // Resume when drain event is emitted + writeStream.once('drain', () => { + console.log('Resuming read stream') + readStream.resume() + }) + } + }) + + readStream.on('end', () => { + writeStream.end() + }) + + writeStream.on('finish', () => { + console.log('File copy completed!') + resolve() + }) + + readStream.on('error', reject) + writeStream.on('error', reject) + }) +} + +// Usage +await copyFileWithBackpressureHandling('large-video.mp4', 'copy-video.mp4') +``` + +### When to Use Streams + +Streams are the best choice when: + +- **Processing large files** (anything over 100MB) +- **Real-time data processing** (logs, live data feeds) +- **Memory is limited** (cloud functions, containers) +- **You need composability** (multiple processing steps) +- **Handling unknown file sizes** (user uploads, network data) + +## Practical Examples and Best Practices + +Let's look at some real-world scenarios and best practices for file operations. + +### Example 1: Processing User Uploads + +```javascript +import { createWriteStream, createReadStream } from 'fs' +import { pipeline } from 'stream/promises' +import { Transform } from 'stream' +import crypto from 'crypto' + +class FileHasher extends Transform { + constructor() { + super() + this.hash = crypto.createHash('sha256') + this.size = 0 + } + + _transform(chunk, encoding, callback) { + this.hash.update(chunk) + this.size += chunk.length + this.push(chunk) // Pass chunk through unchanged + callback() + } + + _flush(callback) { + this.digest = this.hash.digest('hex') + callback() + } +} + +async function saveUserUpload(uploadStream, filename) { + const hasher = new FileHasher() + const writeStream = createWriteStream(`./uploads/${filename}`) + + try { + await pipeline(uploadStream, hasher, writeStream) + + console.log(`File saved: ${filename}`) + console.log(`Size: ${hasher.size} bytes`) + console.log(`SHA256: ${hasher.digest}`) + + return { + filename, + size: hasher.size, + hash: hasher.digest + } + } catch (error) { + console.error('Upload failed:', error.message) + throw error + } +} +``` + +### Example 2: Log File Analysis + +```javascript +import { createReadStream } from 'fs' +import { createInterface } from 'readline' + +async function analyzeLogFile(logPath) { + const fileStream = createReadStream(logPath) + const rl = createInterface({ + input: fileStream, + crlfDelay: Infinity // Handle Windows line endings + }) + + const stats = { + totalLines: 0, + errorLines: 0, + warningLines: 0, + ipAddresses: new Set(), + statusCodes: new Map() + } + + for await (const line of rl) { + stats.totalLines++ + + // Simple log parsing (adjust for your log format) + if (line.includes('ERROR')) { + stats.errorLines++ + } else if (line.includes('WARN')) { + stats.warningLines++ + } + + // Extract IP addresses (simple regex) + const ipMatch = line.match(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/) + if (ipMatch) { + stats.ipAddresses.add(ipMatch[0]) + } + + // Extract HTTP status codes + const statusMatch = line.match(/\s(\d{3})\s/) + if (statusMatch) { + const status = statusMatch[1] + stats.statusCodes.set(status, (stats.statusCodes.get(status) || 0) + 1) + } + } + + console.log('Log Analysis Results:') + console.log(`Total lines: ${stats.totalLines}`) + console.log(`Error lines: ${stats.errorLines}`) + console.log(`Warning lines: ${stats.warningLines}`) + console.log(`Unique IP addresses: ${stats.ipAddresses.size}`) + console.log('Status code distribution:') + + for (const [code, count] of stats.statusCodes) { + console.log(` ${code}: ${count}`) + } + + return stats +} + +// Usage +await analyzeLogFile('./logs/access.log') +``` + +### Best Practices Summary + +1. **Choose the right tool for the job**: + - Small files (< 100MB): Use `fs/promises` + - Large files or unknown sizes: Use streams + - Need precise control: Use file handles + +2. **Always handle errors properly**: + ```javascript + try { + // File operation + } catch (error) { + if (error.code === 'ENOENT') { + console.log('File not found') + } else if (error.code === 'EACCES') { + console.log('Permission denied') + } else { + console.log('Unexpected error:', error.message) + } + } + ``` + +3. **Use appropriate buffer sizes**: + ```javascript + // For large files, use bigger chunks + const stream = createReadStream(path, { + highWaterMark: 64 * 1024 // 64KB chunks + }) + ``` + +4. **Always clean up resources**: + ```javascript + // File handles + try { + const handle = await open(path, 'r') + // Use handle... + } finally { + await handle?.close() + } + + // Streams (or use pipeline()) + stream.on('error', () => stream.destroy()) + ``` + +5. **Consider using pipeline() for stream operations**: + ```javascript + // Better error handling and cleanup + await pipeline(source, transform, destination) + ``` + +## Summary and Conclusion + +We've covered a comprehensive range of file operation techniques in Node.js, from simple promise-based methods to advanced streaming patterns. Here's a quick recap of when to use each approach: + +**Use `fs/promises` when:** +- Files are small to medium-sized (under 100MB) +- You need the entire file content at once +- Simplicity and readability are priorities +- Memory usage isn't a constraint + +**Use file handles when:** +- You need precise control over read/write operations +- Working with specific file positions or ranges +- Building low-level file processing tools +- Performance optimization is critical + +**Use streams when:** +- Processing large files (over 100MB) +- Memory efficiency is important +- You need to compose multiple operations +- Handling real-time or unknown-sized data +- Building scalable applications + +The key to mastering file operations in Node.js is understanding these trade-offs and choosing the right approach for your specific use case. Start with the simple promise-based methods for most scenarios, then move to streams when you need better performance and memory efficiency. + +Remember that file operations are often I/O bound, so proper error handling and resource management are crucial for building robust applications. Whether you're processing user uploads, analyzing log files, or building data pipelines, these patterns will serve you well in your Node.js journey. + +Happy coding, and may your file operations be forever memory-efficient and performant! 🚀 \ No newline at end of file From 42cdbd0d94e5f1345e94a0d6e55fe08860dd010d Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Thu, 9 Oct 2025 19:36:06 +0200 Subject: [PATCH 2/7] feat: progress on the article and added GH style admonitions support --- astro.config.ts | 5 + package.json | 17 +- pnpm-lock.yaml | 127 ++- .../reading-writing-files-nodejs/index.md | 738 +++++++++++++++--- src/plugins/remark-admonitions.ts | 95 +++ src/plugins/utils/remark.ts | 24 + src/styles/global.css | 114 ++- 7 files changed, 992 insertions(+), 128 deletions(-) create mode 100644 src/plugins/remark-admonitions.ts create mode 100644 src/plugins/utils/remark.ts diff --git a/astro.config.ts b/astro.config.ts index d8b9563..34612d4 100644 --- a/astro.config.ts +++ b/astro.config.ts @@ -6,6 +6,8 @@ import tailwindcss from '@tailwindcss/vite' import { defineConfig, fontProviders } from 'astro/config' import expressiveCode from 'astro-expressive-code' import partytown from '@astrojs/partytown' +import remarkDirective from 'remark-directive' /* Handle ::: directives as nodes */ +import { remarkAdmonitions } from './src/plugins/remark-admonitions' /* Add admonitions */ // https://astro.build/config export default defineConfig({ @@ -16,6 +18,9 @@ export default defineConfig({ sitemap(), partytown({ config: { forward: ['dataLayer.push'] } }), ], + markdown: { + remarkPlugins: [remarkDirective, remarkAdmonitions], + }, vite: { plugins: [tailwindcss()], }, diff --git a/package.json b/package.json index 30b44d5..5228749 100644 --- a/package.json +++ b/package.json @@ -30,24 +30,33 @@ "clsx": "^2.1.1", "daisyui": "^5.0.43", "framer-motion": "^12.15.0", + "hastscript": "^9.0.1", "lucide-react": "^0.511.0", + "mdast": "^3.0.0", + "mdast-util-directive": "^3.1.0", + "mdast-util-to-markdown": "^2.1.2", + "mdast-util-to-string": "^4.0.0", "ogl": "^1.0.11", "react": "^19.1.0", "react-dom": "^19.1.0", "rehype-accessible-emojis": "^0.3.2", + "remark-directive": "^4.0.0", "remark-toc": "^9.0.0", "sharp": "^0.34.3", "swiper": "^11.2.10", "tailwind-merge": "^3.3.0", "tailwindcss": "^4.1.8", - "three": "^0.180.0" + "three": "^0.180.0", + "unified": "^11.0.5", + "unist-util-visit": "^5.0.0" }, "packageManager": "pnpm@10.14.0+sha512.ad27a79641b49c3e481a16a805baa71817a04bbe06a38d17e60e2eaee83f6a146c6a688125f5792e48dd5ba30e7da52a5cda4c3992b9ccf333f9ce223af84748", "devDependencies": { - "@eslint/js": "9.35.0", - "@types/three": "^0.180.0", + "@eslint/js": "latest", + "@types/mdast": "^4.0.4", + "@types/three": "latest", "@typescript-eslint/eslint-plugin": "8.37.0", - "@typescript-eslint/parser": "8.43.0", + "@typescript-eslint/parser": "latest", "astro-eslint-parser": "1.2.2", "eslint": "9.35.0", "eslint-config-prettier": "10.1.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dd3e719..977f0fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,9 +56,24 @@ importers: framer-motion: specifier: ^12.15.0 version: 12.15.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + hastscript: + specifier: ^9.0.1 + version: 9.0.1 lucide-react: specifier: ^0.511.0 version: 0.511.0(react@19.1.0) + mdast: + specifier: ^3.0.0 + version: 3.0.0 + mdast-util-directive: + specifier: ^3.1.0 + version: 3.1.0 + mdast-util-to-markdown: + specifier: ^2.1.2 + version: 2.1.2 + mdast-util-to-string: + specifier: ^4.0.0 + version: 4.0.0 ogl: specifier: ^1.0.11 version: 1.0.11 @@ -71,6 +86,9 @@ importers: rehype-accessible-emojis: specifier: ^0.3.2 version: 0.3.2 + remark-directive: + specifier: ^4.0.0 + version: 4.0.0 remark-toc: specifier: ^9.0.0 version: 9.0.0 @@ -89,18 +107,27 @@ importers: three: specifier: ^0.180.0 version: 0.180.0 + unified: + specifier: ^11.0.5 + version: 11.0.5 + unist-util-visit: + specifier: ^5.0.0 + version: 5.0.0 devDependencies: '@eslint/js': - specifier: 9.35.0 + specifier: latest version: 9.35.0 + '@types/mdast': + specifier: ^4.0.4 + version: 4.0.4 '@types/three': - specifier: ^0.180.0 + specifier: latest version: 0.180.0 '@typescript-eslint/eslint-plugin': specifier: 8.37.0 version: 8.37.0(@typescript-eslint/parser@8.43.0(eslint@9.35.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.35.0(jiti@2.4.2))(typescript@5.8.3) '@typescript-eslint/parser': - specifier: 8.43.0 + specifier: latest version: 8.43.0(eslint@9.35.0(jiti@2.4.2))(typescript@5.8.3) astro-eslint-parser: specifier: 1.2.2 @@ -1016,6 +1043,9 @@ packages: '@types/ungap__structured-clone@1.2.0': resolution: {integrity: sha512-ZoaihZNLeZSxESbk9PUAPZOlSpcKx81I1+4emtULDVmBLkYutTcMlCj2K9VNlf9EWODxdO6gkAqEaLorXwZQVA==} + '@types/unist@2.0.11': + resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} + '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} @@ -1273,6 +1303,9 @@ packages: character-entities@2.0.2: resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -1746,9 +1779,18 @@ packages: iron-webcrypto@1.2.1: resolution: {integrity: sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==} + is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + + is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-arrayish@0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} + is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + is-docker@3.0.0: resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1766,6 +1808,9 @@ packages: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} + is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + is-inside-container@1.0.0: resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} engines: {node: '>=14.16'} @@ -1937,6 +1982,9 @@ packages: mdast-util-definitions@6.0.0: resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} + mdast-util-directive@3.1.0: + resolution: {integrity: sha512-I3fNFt+DHmpWCYAT7quoM6lHf9wuqtI+oCOfvILnoicNIqjh5E3dEJWiXuYME2gNe8vl1iMQwyUHa7bgFmak6Q==} + mdast-util-find-and-replace@3.0.2: resolution: {integrity: sha512-Tmd1Vg/m3Xz43afeNxDIhWRtFZgM2VLyaf4vSTYwudTyeuTneoL3qtWMA5jeLyz/O1vDJmmV4QuScFCA2tBPwg==} @@ -1976,6 +2024,10 @@ packages: mdast-util-toc@7.1.0: resolution: {integrity: sha512-2TVKotOQzqdY7THOdn2gGzS9d1Sdd66bvxUyw3aNpWfcPXCLYSJCCgfPy30sEtuzkDraJgqF35dzgmz6xlvH/w==} + mdast@3.0.0: + resolution: {integrity: sha512-xySmf8g4fPKMeC07jXGz971EkLbWAJ83s4US2Tj9lEdnZ142UP5grN73H1Xd3HzrdbU5o9GYYP/y8F9ZSwLE9g==} + deprecated: '`mdast` was renamed to `remark`' + mdn-data@2.12.2: resolution: {integrity: sha512-IEn+pegP1aManZuckezWCO+XZQDplx1366JoVhTpMpBB1sPey/SbveZQUosKiKiGYjg1wH4pMlNgXbCiYgihQA==} @@ -1989,6 +2041,9 @@ packages: micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + micromark-extension-directive@4.0.0: + resolution: {integrity: sha512-/C2nqVmXXmiseSSuCdItCMho7ybwwop6RrrRPk0KbOHW21JKoCldC+8rFOaundDoRBUWBnJJcxeA/Kvi34WQXg==} + micromark-extension-gfm-autolink-literal@2.1.0: resolution: {integrity: sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==} @@ -2199,6 +2254,9 @@ packages: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} + parse-entities@4.0.2: + resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-latin@7.0.0: resolution: {integrity: sha512-mhHgobPPua5kZ98EF4HWiH167JWBfl4pvAIXXdbaVohtK7a6YBOy56kvhCqduqyo/f3yrHFWmqmiMg/BkBkYYQ==} @@ -2335,6 +2393,9 @@ packages: rehype@13.0.2: resolution: {integrity: sha512-j31mdaRFrwFRUIlxGeuPXXKWQxet52RBQRvCmzl5eCefn/KGbomK5GMHNMsOJf55fgo3qw5tST5neDuarDYR2A==} + remark-directive@4.0.0: + resolution: {integrity: sha512-7sxn4RfF1o3izevPV1DheyGDD6X4c9hrGpfdUpm7uC++dqrnJxIZVkk7CoKqcLm0VUMAuOol7Mno3m6g8cfMuA==} + remark-gfm@4.0.1: resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} @@ -3669,6 +3730,8 @@ snapshots: '@types/ungap__structured-clone@1.2.0': {} + '@types/unist@2.0.11': {} + '@types/unist@3.0.3': {} '@types/webxr@0.5.24': {} @@ -4066,6 +4129,8 @@ snapshots: character-entities@2.0.2: {} + character-reference-invalid@2.0.1: {} + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -4605,8 +4670,17 @@ snapshots: iron-webcrypto@1.2.1: {} + is-alphabetical@2.0.1: {} + + is-alphanumerical@2.0.1: + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + is-arrayish@0.3.2: {} + is-decimal@2.0.1: {} + is-docker@3.0.0: {} is-extglob@2.1.1: {} @@ -4617,6 +4691,8 @@ snapshots: dependencies: is-extglob: 2.1.1 + is-hexadecimal@2.0.1: {} + is-inside-container@1.0.0: dependencies: is-docker: 3.0.0 @@ -4751,6 +4827,20 @@ snapshots: '@types/unist': 3.0.3 unist-util-visit: 5.0.0 + mdast-util-directive@3.1.0: + dependencies: + '@types/mdast': 4.0.4 + '@types/unist': 3.0.3 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 + parse-entities: 4.0.2 + stringify-entities: 4.0.4 + unist-util-visit-parents: 6.0.1 + transitivePeerDependencies: + - supports-color + mdast-util-find-and-replace@3.0.2: dependencies: '@types/mdast': 4.0.4 @@ -4875,6 +4965,8 @@ snapshots: unist-util-is: 6.0.0 unist-util-visit: 5.0.0 + mdast@3.0.0: {} + mdn-data@2.12.2: {} merge2@1.4.1: {} @@ -4900,6 +4992,16 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 + micromark-extension-directive@4.0.0: + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.1 + micromark-factory-whitespace: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + parse-entities: 4.0.2 + micromark-extension-gfm-autolink-literal@2.1.0: dependencies: micromark-util-character: 2.1.1 @@ -5185,6 +5287,16 @@ snapshots: dependencies: callsites: 3.1.0 + parse-entities@4.0.2: + dependencies: + '@types/unist': 2.0.11 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.1.0 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + parse-latin@7.0.0: dependencies: '@types/nlcst': 2.0.3 @@ -5324,6 +5436,15 @@ snapshots: rehype-stringify: 10.0.1 unified: 11.0.5 + remark-directive@4.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-directive: 3.1.0 + micromark-extension-directive: 4.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-gfm@4.0.1: dependencies: '@types/mdast': 4.0.4 diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md index ea7332b..ac0daed 100644 --- a/src/content/blog/reading-writing-files-nodejs/index.md +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -10,107 +10,372 @@ tags: ['blog'] File operations are at the heart of most Node.js applications. Whether you're building a web server that serves static assets, processing CSV files, handling user uploads, or working with configuration files, knowing how to efficiently read and write files is absolutely essential. -In this comprehensive guide, we'll explore the modern approaches to file handling in Node.js. We'll start with the simplest methods using promises, then dive into more advanced techniques like streaming and low-level file operations that can handle massive files without breaking your application. +There are several ways to handle file operations in Node.js (callback-based APIs, synchronous methods, and promise-based approaches). While callback and sync methods are still valid and occasionally useful, they're being used less and less these days. The Node.js ecosystem has shifted toward more ergonomic patterns, moving away from callback hell and blocking operations in favor of cleaner `async/await` syntax and non-blocking promise-based APIs. + +In this comprehensive guide, we'll focus on the modern approaches to file handling in Node.js. We'll start with the simplest methods using promises, then dive into more advanced techniques like using file handles and streaming: a clever approach that can handle massive files without breaking your application. Let's dive in and master file operations in Node.js! -## Why File Operations Matter +## Reading and Writing Files with Node.js Promises (fs/promises) -Before we jump into the code, let's talk about why file operations are so crucial in Node.js development. Almost every application you'll build will need to: +The most straightforward way to work with files in modern Node.js is using the `fs/promises` module with functions like `readFile()` and `writeFile()`. This gives us a clean, promise-based API that works beautifully with `async/await`. -- **Read configuration files** (JSON, YAML, environment files) -- **Process user uploads** (images, documents, data files) -- **Generate reports** (CSV exports, PDFs, logs) -- **Cache data** (temporary files, session storage) -- **Serve static content** (HTML, CSS, JavaScript files) +Both `readFile()` and `writeFile()` return promises, and when we use `await`, we give control back to the event loop while the asynchronous file operation completes. When the operation finishes successfully, execution moves to the next line of code. If the operation fails, the promise rejects and throws an error (which is why we need `try/catch` blocks). This non-blocking behavior is what makes Node.js so efficient at handling I/O operations in highly concurrent environments. -The way you handle these operations can make or break your application's performance, especially when dealing with large files or high traffic. +### Reading Files with fs/promises -## Reading and Writing Files with Promises +Here's how you can read a file using the modern promise-based approach: -The most straightforward way to work with files in modern Node.js is using the `fs/promises` module. This gives us a clean, promise-based API that works beautifully with `async/await`. +```javascript {5} +// read-promises.js +import { readFile } from 'node:fs/promises' -### Reading Files with fs/promises +try { + const data = await readFile('config.json', 'utf8') + const config = JSON.parse(data) + console.log('Configuration loaded:', config) +} catch (error) { + console.error('Failed to read config file:', error.message) +} +``` -Here's how you can read a file using the modern promise-based approach: +The `readFile()` function loads the entire file content into memory and returns it as a string (when you specify an encoding like 'utf8') or as a Buffer (when no encoding is specified). -```javascript -import { readFile } from 'fs/promises' +:::tip[Why error handling matters] +Notice how we wrapped our file operation in a `try/catch` block? File operations can fail for many reasons: + +- **ENOENT**: File or directory doesn't exist +- **EACCES**: Permission denied (can't read the file) +- **EISDIR**: Tried to read a directory as a file +- **EMFILE**: Too many open files +- **JSON parsing errors**: If you're reading JSON, `JSON.parse()` can throw (e.g. if the file content isn't valid JSON) + +Always handle these errors gracefully to prevent your application from crashing. The error object includes a `code` property that helps you identify the specific issue and respond appropriately. +::: + +### Writing Files with fs/promises + +Writing files is just as straightforward. This example takes a JavaScript object, converts it to JSON, and saves it to a file: + +```javascript {10} +// write-promises.js +import { writeFile } from 'node:fs/promises' + +try { + const jsonData = JSON.stringify( + { name: 'John Doe', email: 'john@example.com' }, + null, + 2, + ) + await writeFile('user-data.json', jsonData, 'utf8') + console.log('User data saved successfully!') +} catch (error) { + console.error('Failed to save user data:', error.message) + throw error +} +``` + +Notice we're using `try/catch` here as well, because file write operations can fail for various reasons: -async function readConfigFile() { +- **EACCES**: Permission denied (can't write to the directory) +- **ENOSPC**: No space left on device (disk full) +- **EROFS**: Read-only file system +- **EISDIR**: Trying to write to a directory instead of a file +- **ENOTDIR**: Part of the path is not a directory + +### Reading and Writing Binary Files + +Not all files are text-based. Here's a more advanced example showing how to work with binary data by creating and reading WAV audio files: + +#### Writing Binary Data - Generate a WAV file + +```javascript {75} +// write-wav-beep.js +import { writeFile } from 'node:fs/promises' + +// Encode a PCM16 WAV buffer from interleaved Int16 samples +function encodeWavPcm16(samples, sampleRate = 44100, numChannels = 1) { + const bytesPerSample = 2 + const blockAlign = numChannels * bytesPerSample + const byteRate = sampleRate * blockAlign + const dataSize = samples.length * bytesPerSample + + const buf = Buffer.alloc(44 + dataSize) + let o = 0 + + // RIFF header + buf.write('RIFF', o) + o += 4 + buf.writeUInt32LE(36 + dataSize, o) + o += 4 // file size minus 8 + buf.write('WAVE', o) + o += 4 + + // "fmt " chunk (PCM) + buf.write('fmt ', o) + o += 4 + buf.writeUInt32LE(16, o) + o += 4 // chunk size for PCM + buf.writeUInt16LE(1, o) + o += 2 // audioFormat = 1 (PCM) + buf.writeUInt16LE(numChannels, o) + o += 2 + buf.writeUInt32LE(sampleRate, o) + o += 4 + buf.writeUInt32LE(byteRate, o) + o += 4 + buf.writeUInt16LE(blockAlign, o) + o += 2 + buf.writeUInt16LE(16, o) + o += 2 // bitsPerSample + + // "data" chunk + buf.write('data', o) + o += 4 + buf.writeUInt32LE(dataSize, o) + o += 4 + + for (let i = 0; i < samples.length; i++, o += 2) { + buf.writeInt16LE(samples[i], o) + } + return buf +} + +// Make a mono sine wave (Int16 samples) +function makeSine( + durationSec, + freq = 1000, + sampleRate = 44100, + numChannels = 1, + amp = 0.3, +) { + const frames = Math.floor(durationSec * sampleRate) + const samples = new Int16Array(frames * numChannels) + for (let i = 0; i < frames; i++) { + const x = Math.sin(2 * Math.PI * freq * (i / sampleRate)) * amp + const s = Math.max(-1, Math.min(1, x)) + const v = Math.round(s * 32767) + for (let c = 0; c < numChannels; c++) samples[i * numChannels + c] = v + } + return { samples, sampleRate, numChannels } +} + +async function createBeepWav() { try { - const data = await readFile('config.json', 'utf8') - const config = JSON.parse(data) - console.log('Configuration loaded:', config) - return config + const { samples, sampleRate, numChannels } = makeSine(1.0, 1000, 44100, 1) + const wavBuf = encodeWavPcm16(samples, sampleRate, numChannels) + await writeFile('beep.wav', wavBuf) + console.log('Wrote beep.wav') } catch (error) { - console.error('Failed to read config file:', error.message) + console.error('Failed to create WAV file:', error.message) throw error } } // Usage -const config = await readConfigFile() +await createBeepWav() ``` -The `readFile` function loads the entire file content into memory and returns it as a string (when you specify an encoding like 'utf8') or as a Buffer (when no encoding is specified). +When working with binary data like this, we use the Node.js [Buffer](https://nodejs.org/api/buffer.html) object to organize and manipulate the binary data. A Buffer is a fixed-size sequence of bytes that provides a way to work with binary data directly. It's similar to an array of integers, but specifically designed for handling raw binary data efficiently. In our WAV example, we use `Buffer.alloc()` to create a buffer of the required size, then use methods like `writeUInt32LE()` and `writeInt16LE()` to write specific data types at specific positions in little-endian format. If you are curious to find out more about the binary structure of a WAV file, you can check out this excellent page on [Wikipedia](https://en.wikipedia.org/wiki/WAV). -### Writing Files with fs/promises +#### Reading Binary Data - Parse WAV file header -Writing files is just as straightforward: +In this example, we're going to read a WAV file to determine its duration in milliseconds. This requires reading the file content as binary data and then performing binary data processing based on our knowledge of the WAV file structure. We'll parse the WAV header to extract information like sample rate and data size, then calculate the duration using the formula: `(dataSize / byteRate) * 1000`. -```javascript -import { writeFile } from 'fs/promises' +```javascript {51} +// read-wav-duration.js +import { readFile } from 'node:fs/promises' + +function decodeWavHeader(buffer) { + if ( + buffer.toString('ascii', 0, 4) !== 'RIFF' || + buffer.toString('ascii', 8, 12) !== 'WAVE' + ) { + throw new Error('Not a RIFF/WAVE file') + } + + let offset = 12 // after RIFF size + WAVE + let fmt = null + let dataOffset = null + let dataSize = null + + while (offset + 8 <= buffer.length) { + const id = buffer.toString('ascii', offset, offset + 4) + const size = buffer.readUInt32LE(offset + 4) + const payload = offset + 8 + + if (id === 'fmt ') { + fmt = { + audioFormat: buffer.readUInt16LE(payload + 0), + numChannels: buffer.readUInt16LE(payload + 2), + sampleRate: buffer.readUInt32LE(payload + 4), + byteRate: buffer.readUInt32LE(payload + 8), + blockAlign: buffer.readUInt16LE(payload + 12), + bitsPerSample: buffer.readUInt16LE(payload + 14), + } + } else if (id === 'data') { + dataOffset = payload + dataSize = size + break // found PCM payload + } + + // Chunks are padded to even byte boundaries + offset = payload + size + (size % 2) + } + + if (!fmt || dataOffset == null) throw new Error('Missing fmt or data chunk') + return { ...fmt, dataOffset, dataSize } +} + +async function getWavDuration(filePath) { + if (!filePath) { + throw new Error('File path is required') + } -async function saveUserData(userData) { try { - const jsonData = JSON.stringify(userData, null, 2) - await writeFile('user-data.json', jsonData, 'utf8') - console.log('User data saved successfully!') + const buf = await readFile(filePath) + const hdr = decodeWavHeader(buf) + + // duration (ms) = (dataSize / byteRate) * 1000 + const durationMs = Math.round((hdr.dataSize / hdr.byteRate) * 1000) + console.log(`Duration: ${durationMs}ms`) + return durationMs } catch (error) { - console.error('Failed to save user data:', error.message) + console.error('Failed to read WAV file:', error.message) throw error } } -// Usage -await saveUserData({ name: 'John Doe', email: 'john@example.com' }) +// Usage (file path must be provided) +const filePath = process.argv[2] +if (!filePath) { + console.error('Usage: node read-wav-duration.js ') + process.exit(1) +} +await getWavDuration(filePath) ``` -### Reading and Writing Binary Files +You can try out this example with the beep WAV files we created in the previous example or, if you need some longer free WAV files, you can check out a website with free audio samples like [Zapsplat](https://www.zapsplat.com/). + +:::note[Binary file complexity] +This is a very simple example showing basic binary manipulation using only Node.js built-ins. We can handle the WAV format manually here because we're creating a minimal, single-channel PCM file with known parameters. + +For real-world audio applications, you'd typically want to use a comprehensive library like [Howler.js](https://howlerjs.com/) that handles the complexity of multiple audio formats and advanced audio features. The same principle applies to other binary formats - use specialized libraries when available, but understanding the underlying binary operations helps you debug issues and work with custom formats. +::: + +### Concurrent File Operations with Promises + +One of the great advantages of promise-based file operations is the ability to read and write multiple files concurrently. Here's how you can do it: + +#### Reading Multiple Files Concurrently + +```javascript {15, 19-20} +// read-multiple-files.js +import { readFile } from 'node:fs/promises' + +const configFiles = [ + 'config/database.json', + 'config/api.json', + 'config/logging.json', + 'config/feature-flags.json', +] -Not all files are text-based. Here's how you handle binary files like images: +try { + // Read all files concurrently and parse JSON + // Note that we are not using await here, so the reads happen concurrently + const promises = configFiles.map((file) => + readFile(file, 'utf8').then((content) => JSON.parse(content)), + ) + + // Here we are using await, so we wait for all reads to complete before proceeding + const [databaseConfig, apiConfig, loggingConfig, featureFlagsConfig] = + await Promise.all(promises) + + console.log(`Successfully loaded config files`, { + databaseConfig, + apiConfig, + loggingConfig, + featureFlagsConfig, + }) +} catch (error) { + console.error('Failed to read config files:', error.message) +} +``` + +This concurrent approach provides a significant performance improvement. If we used multiple `await` statements (one for each file read), we would be processing the files sequentially. If each file read takes 10ms, we wouldn't be completing the operation in less than 40ms. With concurrent reads - by starting all promises first and then awaiting all of them with `Promise.all()` - we'll most likely be able to read all four files in around 10ms (roughly the time of the slowest individual read). + +#### Writing Multiple Files Concurrently ```javascript -import { readFile, writeFile } from 'fs/promises' +// write-multiple-files.js +import { writeFile } from 'node:fs/promises' + +async function generateReports(data) { + const reports = [ + { + filename: 'reports/daily-summary.json', + content: JSON.stringify(data.daily, null, 2), + }, + { + filename: 'reports/weekly-summary.json', + content: JSON.stringify(data.weekly, null, 2), + }, + { + filename: 'reports/monthly-summary.json', + content: JSON.stringify(data.monthly, null, 2), + }, + { + filename: 'reports/yearly-summary.json', + content: JSON.stringify(data.yearly, null, 2), + }, + ] -async function copyImage(sourcePath, destinationPath) { try { - // Read binary data (no encoding specified) - const imageData = await readFile(sourcePath) - - // Write binary data - await writeFile(destinationPath, imageData) + // Write all files concurrently + const promises = reports.map(({ filename, content }) => + writeFile(filename, content, 'utf8'), + ) - console.log('Image copied successfully!') - console.log(`File size: ${imageData.length} bytes`) + await Promise.all(promises) + console.log(`Successfully generated ${reports.length} report files`) } catch (error) { - console.error('Failed to copy image:', error.message) + console.error('Failed to write report files:', error.message) throw error } } // Usage -await copyImage('original.jpg', 'copy.jpg') +const reportData = { + daily: { sales: 1000, visitors: 250 }, + weekly: { sales: 7000, visitors: 1750 }, + monthly: { sales: 30000, visitors: 7500 }, + yearly: { sales: 365000, visitors: 91250 }, +} + +await generateReports(reportData) ``` +:::tip[Promise.all() vs Promise.allSettled() - Choose the Right Tool] +In these examples, we use `Promise.all()` because if any config file fails to load or any report fails to write, we can't continue - all operations are required for the application to function properly. + +However, if you can tolerate one or more errors (like loading optional config files or processing a batch of user uploads), it's much better to use `Promise.allSettled()`. + +**Key differences:** + +- **`Promise.all()`**: Fails fast - rejects immediately when any promise rejects +- **`Promise.allSettled()`**: Waits for all promises to complete, regardless of whether they succeed or fail + +Learn more: [Promise.all() on MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all) | [Promise.allSettled() on MDN](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled) +::: + ### Working with Directories You'll often need to work with directories too: -```javascript -import { readdir, mkdir, stat } from 'fs/promises' -import { join } from 'path' +```javascript {8,11,14,15,17} +// process-directory.js +import { readdir, mkdir, stat } from 'node:fs/promises' +import { join } from 'node:path' async function processDirectory(dirPath) { try { @@ -139,7 +404,107 @@ async function processDirectory(dirPath) { await processDirectory('./uploads') ``` -## The Memory Problem: Why Promises Aren't Always Enough +This example demonstrates several important Node.js file system operations working together. Let's break down what each function does: + +- **`mkdir('processed', { recursive: true })`** creates a new directory called "processed". The `recursive: true` option means it will create any missing parent directories in the path, similar to `mkdir -p` in Unix systems. If the directory already exists, this won't throw an error. By the way, if you used the [`mkdirp`](https://www.npmjs.com/package/mkdirp) package in the past, you can now use this built-in functionality instead. + +- **`readdir(dirPath)`** reads the contents of a directory and returns an array of file and directory names. This is the asynchronous equivalent of `ls` or `dir` commands, giving you a list of everything in the specified directory. + +- **`stat(filePath)`** retrieves detailed information about a file or directory, including size, creation time, modification time, and whether it's a file or directory. The returned stats object provides methods like `isFile()`, `isDirectory()`, and properties like `size`. + +- **`join(dirPath, file)`** from the `path` module safely combines directory and file names into a complete path, handling path separators correctly across different operating systems (Windows uses `\`, Unix-like systems use `/`). + +The `for...of` loop processes each item in the directory sequentially. If you wanted to process files concurrently for better performance, you could replace this with `Promise.all()` and `files.map()` as we showed in earlier examples. + +#### Referencing Files Relative to Your Script + +When working with files in your Node.js applications, you often need to reference files relative to your current script's location. Modern Node.js provides `import.meta.dirname` (similar to the old `__dirname`) to get the directory of the current module: + +```javascript {8-9,19-20} +// read-config-relative.js +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' + +async function loadConfig() { + try { + // Read config.json from the same directory as this script + const configPath = join(import.meta.dirname, 'config.json') + const configData = await readFile(configPath, 'utf8') + return JSON.parse(configData) + } catch (error) { + console.error('Failed to load config:', error.message) + throw error + } +} + +async function loadTemplate() { + // Read template from a subdirectory relative to this script + const templatePath = join(import.meta.dirname, 'templates', 'email.html') + return await readFile(templatePath, 'utf8') +} + +// Usage +const config = await loadConfig() +const emailTemplate = await loadTemplate() +``` + +This approach is much more reliable than using relative paths like `'./config.json'` because `import.meta.dirname` always refers to the directory containing your script file, regardless of where the Node.js process was started from. This prevents the common issue where your script works when run from its own directory but fails when run from elsewhere. + +### Async vs Sync File Operations: When to Use Which + +Before diving into more advanced file handling techniques, let's compare the promise-based approaches we've seen with the synchronous alternatives Node.js provides. + +Node.js also offers synchronous versions of file operations like `readFileSync()` and `writeFileSync()`: + +```javascript {6,17} +// sync-file-operations.js +import { readFileSync, writeFileSync } from 'node:fs' + +// Synchronous file reading +try { + const data = readFileSync('config.json', 'utf8') + const config = JSON.parse(data) + console.log('Configuration loaded:', config) +} catch (error) { + console.error('Failed to read config file:', error.message) +} + +// Synchronous file writing +try { + const userData = { name: 'John Doe', email: 'john@example.com' } + const jsonData = JSON.stringify(userData, null, 2) + writeFileSync('user-data.json', jsonData, 'utf8') + console.log('User data saved successfully!') +} catch (error) { + console.error('Failed to save user data:', error.message) +} +``` + +Notice that when using `readFileSync()` and `writeFileSync()` we are not using `await`. This is because these methods are synchronous: they block the event loop until the operation is completed, which means no other JavaScript code is executed while the file read/write operation is in progress. + +**My recommendation: I generally prefer to avoid synchronous file functions entirely and always go for the async approach for consistency.** This keeps your codebase uniform and prevents accidentally blocking the event loop in unexpected places. This makes even more sense since for a few years Node.js has added support for **top-level await** (being able to use `await` outside an async function), so using asynchronous file operations is just as easy as using synchronous ones. + +However, if you care about extreme performance, there are specific cases where you might prefer the sync methods. Sync methods can occasionally be faster because they don't need to give back control to the event loop and, for example in the case of reading a file, the content of the file is immediately available once the filesystem operation completes without having to wait for the event loop to capture the event and give back control to our code. One case that comes to mind is when writing **non-concurrent CLI apps or scripts** that need to read or write a file before continuing with the next operation. + +:::danger[Never Use Sync Methods in Concurrent Environments] +**Do not use sync methods in concurrent environments such as web servers.** While you read or write a file synchronously, you will be blocking the event loop, which means that no other user requests will be processed during that time. + +This can cause noticeable delays for users - if a file operation takes 100ms, every user trying to access your server during that time will experience a 100ms delay. In a web application, this is unacceptable. + +**Safe for sync methods:** + +- CLI tools and scripts +- Build processes +- Non-concurrent applications + +**Never use sync methods in:** + +- Web servers (Express, Fastify, etc.) +- Real-time applications +- Any concurrent environment +::: + +## Working with Large Files: Memory Considerations The promise-based approach we've seen so far is perfect for small to medium-sized files. However, there's a significant limitation: **everything gets loaded into memory at once**. @@ -147,13 +512,13 @@ Imagine you're trying to read a 2GB log file using `readFile()`. Your Node.js pr 1. **Out of memory errors** - Your application might crash 2. **Poor performance** - High memory usage affects other operations -3. **Blocking behavior** - Large file operations can block your event loop ### Understanding Node.js Buffer Limits Node.js has built-in limits on buffer sizes to prevent applications from consuming too much memory. You can check these limits: ```javascript +// check-buffer-limits.js // Check the maximum buffer size console.log('Max buffer size:', Buffer.constants.MAX_LENGTH) console.log('Max string length:', Buffer.constants.MAX_STRING_LENGTH) @@ -166,7 +531,8 @@ console.log('Max string length:', Buffer.constants.MAX_STRING_LENGTH) If you try to read a file larger than these limits using `readFile()`, you'll get an error: ```javascript -import { readFile } from 'fs/promises' +// handle-large-file-error.js +import { readFile } from 'node:fs/promises' try { // This will fail if the file is larger than the buffer limit @@ -178,6 +544,8 @@ try { } ``` +So does this mean that Node.js can't handle big files?! Of course not, we just need to use different tools to do that! The trick is to make sure we don't load ALL the data into memory in one go, but we process the data in smaller incremental chunks! + ### When to Use Promise-Based Methods Promise-based file operations are great when: @@ -187,14 +555,17 @@ Promise-based file operations are great when: - **Simplicity is important** (rapid prototyping, simple scripts) - **Memory usage isn't a concern** (plenty of RAM available) -## Low-Level File Operations with File Handles +## Advanced Node.js File Operations with File Handles When you need more control over file operations, Node.js provides lower-level APIs using file handles. This approach allows you to read and write files incrementally, giving you fine-grained control over memory usage. +If you have ever read data from a file or written data into a file in a low-level language such as C, this approach will seem familiar. + ### Opening and Working with File Handles -```javascript -import { open } from 'fs/promises' +```javascript {9,18,40} +// read-file-chunks.js +import { open } from 'node:fs/promises' async function readFileInChunks(filePath) { let fileHandle @@ -226,7 +597,6 @@ async function readFileInChunks(filePath) { } console.log(`Total bytes read: ${totalBytesRead}`) - } catch (error) { console.error('Error reading file:', error.message) throw error @@ -242,37 +612,69 @@ async function readFileInChunks(filePath) { await readFileInChunks('large-file.txt') ``` +This example demonstrates the core concepts of working with file handles in Node.js. The `open()` function creates a file handle that represents our connection to the file, similar to how you would open a file in C or other low-level languages. We pass `'r'` as the second parameter to indicate we want to open the file for reading. + +The `read()` method allows us to read a specific number of bytes from a specific position in the file. In our loop, we read 1KB chunks at a time, processing each chunk before moving to the next. This approach keeps memory usage low and predictable, regardless of the file size. + +Most importantly, we need to make sure we clean up resources, which is why we use a `finally` block. This way, the cleanup code (calling `fileHandle.close()`) is executed whether everything goes well or if there's an error. This allows us to retain a clean state and lets Node.js reclaim resources that aren't needed anymore. Failing to close file handles can lead to resource leaks and eventually cause your application to run out of available file descriptors. + ### Writing Files Incrementally -```javascript -import { open } from 'fs/promises' +Let's say we want to create a file that contains 1 million unique voucher codes. If we prepopulate all the data in memory, that would require a significant amount of memory. Instead, we can generate vouchers in small chunks and append them to a file as we go: + +```javascript {20,42,56} +// generate-voucher-codes.js +import { open } from 'node:fs/promises' + +// This is for demonstration purposes only. +// Ideally, voucher codes should be generated using a secure random generator +function generateVoucherCode() { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + let result = '' + for (let i = 0; i < 8; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)) + } + return result +} -async function writeFileInChunks(filePath, data) { +async function generateVoucherFile(filePath, totalVouchers) { let fileHandle try { // Open file for writing (creates if doesn't exist) fileHandle = await open(filePath, 'w') - const chunkSize = 1024 - const buffer = Buffer.from(data, 'utf8') + const chunkSize = 1000 // Generate 1000 vouchers per chunk let position = 0 + let vouchersGenerated = 0 + + while (vouchersGenerated < totalVouchers) { + // Generate a chunk of voucher codes + const vouchersInThisChunk = Math.min( + chunkSize, + totalVouchers - vouchersGenerated, + ) + const vouchers = [] + + for (let i = 0; i < vouchersInThisChunk; i++) { + vouchers.push(generateVoucherCode()) + } - while (position < buffer.length) { - const remainingBytes = buffer.length - position - const bytesToWrite = Math.min(chunkSize, remainingBytes) + // Convert to buffer and write to file + const chunk = vouchers.join('\n') + '\n' + const buffer = Buffer.from(chunk, 'utf8') - const chunk = buffer.subarray(position, position + bytesToWrite) - const result = await fileHandle.write(chunk, 0, bytesToWrite, position) + const result = await fileHandle.write(buffer, 0, buffer.length, position) - console.log(`Wrote ${result.bytesWritten} bytes at position ${position}`) position += result.bytesWritten - } + vouchersGenerated += vouchersInThisChunk - console.log('File writing completed!') + console.log(`Generated ${vouchersGenerated}/${totalVouchers} vouchers`) + } + console.log(`Successfully generated ${totalVouchers} voucher codes!`) } catch (error) { - console.error('Error writing file:', error.message) + console.error('Error generating voucher file:', error.message) throw error } finally { if (fileHandle) { @@ -281,11 +683,18 @@ async function writeFileInChunks(filePath, data) { } } -// Usage -const largeText = 'A'.repeat(10000) // 10KB of 'A' characters -await writeFileInChunks('output.txt', largeText) +// Usage - Generate 1 million voucher codes +await generateVoucherFile('voucher-codes.txt', 1000000) ``` +This example shows how to efficiently generate and write large amounts of data without overwhelming memory. The `open()` function creates a file handle with the `'w'` flag, which opens the file for writing (creating it if it doesn't exist, or truncating it if it does). + +The `write()` method takes a buffer and writes it to the file at a specific position. In our example, we keep track of the position manually and increment it after each write operation. This ensures that each chunk of voucher codes is written sequentially to the file without overwriting previous data. + +The beauty of this approach is that we only keep 1,000 voucher codes in memory at any given time, regardless of whether we're generating 1 million or 10 million codes. This keeps memory usage constant and predictable. + +Just like with reading, we use a `finally` block to ensure the file handle is properly closed with `fileHandle.close()`, preventing resource leaks and allowing Node.js to clean up the file descriptor. + ### Why File Handles Are Low-Level While file handles give you precise control, they require you to manage a lot of details manually: @@ -297,10 +706,26 @@ While file handles give you precise control, they require you to manage a lot of This low-level approach is powerful but verbose. For most use cases, there's a better solution: **streams**. -## Streams: The Best of Both Worlds +## Node.js Streams: Memory-Efficient File Processing Streams provide the memory efficiency of low-level file operations with a much more ergonomic API. They're perfect for processing large files without loading everything into memory. +:::tip[Want to Master Node.js Streams?] +We're giving away for free an entire chapter from our book "Node.js Design Patterns" dedicated to learning Streams! + +**Chapter 6: Coding with Streams** - 80 pages packed with practical examples, real-world insights, and powerful design patterns to help you write faster, leaner, and more scalable Node.js code. + +In this chapter, you'll learn: + +- How streams fit into the Node.js philosophy +- The anatomy of readable, writable, and transform streams +- How to handle backpressure and avoid memory issues +- Real-world streaming patterns for composability and performance +- Tips for error handling, concurrency, and combining streams effectively + +[Get your free chapter here →](/#free-chapter) +::: + ### Understanding Stream Advantages Streams offer several key benefits: @@ -315,13 +740,14 @@ Streams offer several key benefits: Here's how to read a file using a readable stream: ```javascript -import { createReadStream } from 'fs' -import { pipeline } from 'stream/promises' +// process-large-file-stream.js +import { createReadStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' async function processLargeFile(filePath) { const readStream = createReadStream(filePath, { encoding: 'utf8', - highWaterMark: 16 * 1024 // 16KB chunks + highWaterMark: 16 * 1024, // 16KB chunks }) let lineCount = 0 @@ -355,14 +781,15 @@ await processLargeFile('huge-log-file.txt') Writing with streams is equally straightforward: ```javascript -import { createWriteStream } from 'fs' -import { pipeline } from 'stream/promises' -import { Readable } from 'stream' +// generate-large-file.js +import { createWriteStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' +import { Readable } from 'node:stream' async function generateLargeFile(filePath, numberOfLines) { const writeStream = createWriteStream(filePath, { encoding: 'utf8', - highWaterMark: 64 * 1024 // 64KB buffer + highWaterMark: 64 * 1024, // 64KB buffer }) // Create a readable stream that generates data @@ -374,7 +801,7 @@ async function generateLargeFile(filePath, numberOfLines) { } else { this.push(null) // End the stream } - } + }, }) try { @@ -396,9 +823,10 @@ await generateLargeFile('generated-data.txt', 1000000) One of the most powerful features of streams is the ability to compose them. Here's an example that reads a CSV file, processes it, and writes the results: ```javascript -import { createReadStream, createWriteStream } from 'fs' -import { pipeline } from 'stream/promises' -import { Transform } from 'stream' +// csv-processor.js +import { createReadStream, createWriteStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' +import { Transform } from 'node:stream' // Custom transform stream to process CSV data class CSVProcessor extends Transform { @@ -449,7 +877,7 @@ async function processCSVFile(inputPath, outputPath) { await pipeline( createReadStream(inputPath, { encoding: 'utf8' }), new CSVProcessor(), - createWriteStream(outputPath, { encoding: 'utf8' }) + createWriteStream(outputPath, { encoding: 'utf8' }), ) console.log('CSV processing completed!') @@ -468,7 +896,8 @@ await processCSVFile('users.csv', 'processed-users.json') Streams automatically handle backpressure - the situation where data is being produced faster than it can be consumed. Here's how you can monitor and control it: ```javascript -import { createReadStream, createWriteStream } from 'fs' +// backpressure-handling.js +import { createReadStream, createWriteStream } from 'node:fs' function copyFileWithBackpressureHandling(source, destination) { return new Promise((resolve, reject) => { @@ -519,17 +948,18 @@ Streams are the best choice when: - **You need composability** (multiple processing steps) - **Handling unknown file sizes** (user uploads, network data) -## Practical Examples and Best Practices +## Real-World Node.js File Operations: Examples and Best Practices Let's look at some real-world scenarios and best practices for file operations. ### Example 1: Processing User Uploads ```javascript -import { createWriteStream, createReadStream } from 'fs' -import { pipeline } from 'stream/promises' -import { Transform } from 'stream' -import crypto from 'crypto' +// process-upload.js +import { createWriteStream, createReadStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' +import { Transform } from 'node:stream' +import crypto from 'node:crypto' class FileHasher extends Transform { constructor() { @@ -565,7 +995,7 @@ async function saveUserUpload(uploadStream, filename) { return { filename, size: hasher.size, - hash: hasher.digest + hash: hasher.digest, } } catch (error) { console.error('Upload failed:', error.message) @@ -577,14 +1007,15 @@ async function saveUserUpload(uploadStream, filename) { ### Example 2: Log File Analysis ```javascript -import { createReadStream } from 'fs' -import { createInterface } from 'readline' +// analyze-logs.js +import { createReadStream } from 'node:fs' +import { createInterface } from 'node:readline' async function analyzeLogFile(logPath) { const fileStream = createReadStream(logPath) const rl = createInterface({ input: fileStream, - crlfDelay: Infinity // Handle Windows line endings + crlfDelay: Infinity, // Handle Windows line endings }) const stats = { @@ -592,7 +1023,7 @@ async function analyzeLogFile(logPath) { errorLines: 0, warningLines: 0, ipAddresses: new Set(), - statusCodes: new Map() + statusCodes: new Map(), } for await (const line of rl) { @@ -639,13 +1070,22 @@ await analyzeLogFile('./logs/access.log') ### Best Practices Summary -1. **Choose the right tool for the job**: - - Small files (< 100MB): Use `fs/promises` - - Large files or unknown sizes: Use streams - - Need precise control: Use file handles +1. **Choose the right approach for your use case**: + - **Small files (< 100MB)**: Use `fs/promises` with `readFile()` and `writeFile()` + - **Large files or unknown sizes**: Use streams with `createReadStream()` and `createWriteStream()` + - **Need precise control**: Use file handles with `open()` and manual read/write operations + - **CLI tools and scripts**: Sync methods (`readFileSync()`, `writeFileSync()`) are acceptable + - **Web servers and concurrent apps**: Never use sync methods - always use async approaches + +2. **Leverage concurrency for better performance**: + - Use `Promise.all()` when all operations must succeed + - Use `Promise.allSettled()` when you can tolerate partial failures + - Read/write multiple files concurrently when possible + +3. **Always handle errors properly**: -2. **Always handle errors properly**: ```javascript + // error-handling-example.js try { // File operation } catch (error) { @@ -653,22 +1093,28 @@ await analyzeLogFile('./logs/access.log') console.log('File not found') } else if (error.code === 'EACCES') { console.log('Permission denied') + } else if (error.code === 'ENOSPC') { + console.log('No space left on device') } else { console.log('Unexpected error:', error.message) } } ``` -3. **Use appropriate buffer sizes**: +4. **Use appropriate buffer sizes for streams**: + ```javascript + // stream-buffer-size.js // For large files, use bigger chunks const stream = createReadStream(path, { - highWaterMark: 64 * 1024 // 64KB chunks + highWaterMark: 64 * 1024, // 64KB chunks }) ``` -4. **Always clean up resources**: +5. **Always clean up resources**: + ```javascript + // resource-cleanup.js // File handles try { const handle = await open(path, 'r') @@ -681,8 +1127,14 @@ await analyzeLogFile('./logs/access.log') stream.on('error', () => stream.destroy()) ``` -5. **Consider using pipeline() for stream operations**: +6. **Use modern JavaScript patterns**: + - Array destructuring for concurrent operations + - Promise chaining for data transformation + - Proper error handling with `try/catch` blocks + +7. **Consider using pipeline() for stream operations**: ```javascript + // pipeline-example.js // Better error handling and cleanup await pipeline(source, transform, destination) ``` @@ -691,27 +1143,87 @@ await analyzeLogFile('./logs/access.log') We've covered a comprehensive range of file operation techniques in Node.js, from simple promise-based methods to advanced streaming patterns. Here's a quick recap of when to use each approach: -**Use `fs/promises` when:** +**Use `fs/promises` (async methods) when:** + - Files are small to medium-sized (under 100MB) - You need the entire file content at once -- Simplicity and readability are priorities -- Memory usage isn't a constraint +- Working in concurrent environments (web servers, APIs) +- You want consistency across your codebase +- Memory usage isn't a critical constraint + +**Use sync methods (`readFileSync`, `writeFileSync`) when:** + +- Building CLI tools or scripts +- Working in non-concurrent environments +- You need to complete file operations before continuing +- Extreme performance is critical for simple operations + +**Use concurrent operations when:** + +- Reading/writing multiple files +- Operations can be performed in parallel +- You want to maximize I/O efficiency **Use file handles when:** + - You need precise control over read/write operations - Working with specific file positions or ranges - Building low-level file processing tools - Performance optimization is critical **Use streams when:** + - Processing large files (over 100MB) - Memory efficiency is important - You need to compose multiple operations - Handling real-time or unknown-sized data - Building scalable applications +- Working with binary data formats + +**Key principles for modern Node.js file operations:** + +1. **Prefer async by default** - Use promise-based methods unless you have a specific reason to use sync methods +2. **Leverage concurrency** - Process multiple files simultaneously when possible +3. **Handle errors gracefully** - Always use proper error handling with specific error codes +4. **Choose the right tool** - Match your approach to your file size and use case +5. **Use modern JavaScript** - Take advantage of array destructuring, promise chaining, and async/await + +The key to mastering file operations in Node.js is understanding these trade-offs and choosing the right approach for your specific use case. Start with the simple promise-based methods for most scenarios, leverage concurrency when processing multiple files, and move to streams when you need better performance and memory efficiency for large files. + +Remember that file operations are often I/O bound, so proper error handling and resource management are crucial for building robust applications. Whether you're building web servers, processing user uploads, analyzing log files, or creating CLI tools, these patterns will serve you well in your Node.js journey. + +## Frequently Asked Questions + +### What's the difference between sync and async file operations in Node.js? + +Synchronous file operations like `readFileSync()` block the event loop until the operation completes, meaning no other JavaScript code can execute during that time. Asynchronous operations like `readFile()` don't block the event loop, allowing other code to run while the file operation happens in the background. Always use async operations in web servers and concurrent applications to avoid blocking other requests. + +### How do I handle large files efficiently in Node.js? + +For large files (over 100MB), avoid `readFile()` and `writeFile()` as they load everything into memory. Instead, use streams (`createReadStream()`, `createWriteStream()`) or file handles with manual chunking. Streams are generally the best choice as they provide automatic backpressure handling and composability. + +### When should I use streams vs promises for file operations? + +Use promises (`fs/promises`) for small to medium files where you need the entire content at once (JSON configs, small text files). Use streams for large files, real-time processing, or when you need memory efficiency. Streams are also better when you need to transform data or chain multiple operations together. + +### What error codes should I handle in Node.js file operations? + +Common error codes include: `ENOENT` (file doesn't exist), `EACCES` (permission denied), `EISDIR` (tried to read directory as file), `ENOSPC` (disk full), `EMFILE` (too many open files). You should handle these specifically rather than just catching generic errors. + +### Can I use top-level await with Node.js file operations? + +Yes! Modern Node.js supports top-level await in ES modules, so you can use `await readFile()` directly without wrapping it in an async function. This makes async file operations as convenient as sync ones, which is another reason to prefer the async approach. + +### How do I process multiple files concurrently in Node.js? + +Use `Promise.all()` or `Promise.allSettled()` with an array of promises to process multiple files simultaneously. For example: `await Promise.all(filenames.map(name => readFile(name)))`. This is much faster than processing files sequentially, especially for I/O-bound operations. + +--- + +## Take Your Node.js Skills to the Next Level -The key to mastering file operations in Node.js is understanding these trade-offs and choosing the right approach for your specific use case. Start with the simple promise-based methods for most scenarios, then move to streams when you need better performance and memory efficiency. +If you found value in this comprehensive guide to file operations, you'll love **Node.js Design Patterns** - the definitive resource designed to elevate Node.js developers from junior to senior level. -Remember that file operations are often I/O bound, so proper error handling and resource management are crucial for building robust applications. Whether you're processing user uploads, analyzing log files, or building data pipelines, these patterns will serve you well in your Node.js journey. +This book dives deep into the patterns, techniques, and best practices that separate good Node.js code from great Node.js code. From fundamental concepts like the ones covered in this article to advanced architectural patterns for building scalable applications, it provides the knowledge you need to write professional, maintainable Node.js code. -Happy coding, and may your file operations be forever memory-efficient and performant! 🚀 \ No newline at end of file +**Ready to master Node.js?** Visit our [homepage](/) to discover how Node.js Design Patterns can accelerate your development journey and help you build better applications with confidence. diff --git a/src/plugins/remark-admonitions.ts b/src/plugins/remark-admonitions.ts new file mode 100644 index 0000000..423f057 --- /dev/null +++ b/src/plugins/remark-admonitions.ts @@ -0,0 +1,95 @@ +import type { Parent, PhrasingContent, Root } from 'mdast' +import type { LeafDirective, TextDirective } from 'mdast-util-directive' +import { directiveToMarkdown } from 'mdast-util-directive' +import { toMarkdown } from 'mdast-util-to-markdown' +import { toString as mdastToString } from 'mdast-util-to-string' +import type { Plugin } from 'unified' +import { visit } from 'unist-util-visit' +import { h, isNodeDirective } from './utils/remark' + +type AdmonitionType = 'tip' | 'note' | 'important' | 'caution' | 'warning' + +// Supported admonition types +const Admonitions = new Set([ + 'tip', + 'note', + 'important', + 'caution', + 'warning', +]) + +/** Checks if a string is a supported admonition type. */ +function isAdmonition(s: string): s is AdmonitionType { + return Admonitions.has(s as AdmonitionType) +} + +/** + * From Astro Starlight: + * Transforms directives not supported back to original form as it can break user content and result in 'broken' output. + */ +function transformUnhandledDirective( + node: LeafDirective | TextDirective, + index: number, + parent: Parent, +) { + const textNode = { + type: 'text', + value: toMarkdown(node, { extensions: [directiveToMarkdown()] }), + } as const + if (node.type === 'textDirective') { + parent.children[index] = textNode + } else { + parent.children[index] = { + children: [textNode], + type: 'paragraph', + } + } +} + +export const remarkAdmonitions: Plugin<[], Root> = () => (tree) => { + visit(tree, (node, index, parent) => { + if (!parent || index === undefined || !isNodeDirective(node)) return + if (node.type === 'textDirective' || node.type === 'leafDirective') { + transformUnhandledDirective(node, index, parent) + return + } + + const admonitionType = node.name + if (!isAdmonition(admonitionType)) return + + let title: string = admonitionType + let titleNode: PhrasingContent[] = [{ type: 'text', value: title }] + + // Check if there's a custom title + const firstChild = node.children[0] + if ( + firstChild?.type === 'paragraph' && + firstChild.data && + 'directiveLabel' in firstChild.data && + firstChild.children.length > 0 + ) { + titleNode = firstChild.children + title = mdastToString(firstChild.children) + // The first paragraph contains a custom title, we can safely remove it. + node.children.splice(0, 1) + } + + // Do not change prefix to AD, ADM, or similar, adblocks will block the content inside. + const admonition = h( + 'aside', + { + 'aria-label': title, + class: 'admonition', + 'data-admonition-type': admonitionType, + }, + [ + h('p', { class: 'admonition-title', 'aria-hidden': 'true' }, [ + ...titleNode, + ]), + h('div', { class: 'admonition-content' }, node.children), + ], + ) + + parent.children[index] = admonition + }) +} diff --git a/src/plugins/utils/remark.ts b/src/plugins/utils/remark.ts new file mode 100644 index 0000000..b1a807e --- /dev/null +++ b/src/plugins/utils/remark.ts @@ -0,0 +1,24 @@ +import { h as _h, type Properties } from 'hastscript' +import type { Node, Paragraph as P } from 'mdast' +import type { Directives } from 'mdast-util-directive' + +/** Checks if a node is a directive. */ +export function isNodeDirective(node: Node): node is Directives { + return ( + node.type === 'containerDirective' || + node.type === 'leafDirective' || + node.type === 'textDirective' + ) +} + +/** From Astro Starlight: Function that generates an mdast HTML tree ready for conversion to HTML by rehype. */ +// biome-ignore lint/suspicious/noExplicitAny: +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function h(el: string, attrs: Properties = {}, children: any[] = []): P { + const { properties, tagName } = _h(el, attrs) + return { + children, + data: { hName: tagName, hProperties: properties }, + type: 'paragraph', + } +} diff --git a/src/styles/global.css b/src/styles/global.css index 41f2ca6..be2d141 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -39,8 +39,10 @@ --color-accent-content: oklch(98% 0.003 247.858); --color-neutral: oklch(44% 0.043 257.281); --color-neutral-content: oklch(98% 0.003 247.858); - --color-info: oklch(95% 0.026 236.824); - --color-info-content: oklch(29% 0.066 243.157); + --color-info: oklch(0.7 0.1 223); + --color-info-content: oklch(0.4137 0.1 223); + --color-tip: oklch(0.8133 0.1 113.29); + --color-tip-content: oklch(0.3133 0.1 113.29); --color-success: oklch(79% 0.209 151.711); --color-success-content: oklch(37% 0.077 168.94); --color-warning: oklch(83% 0.128 66.29); @@ -92,15 +94,14 @@ --color-neutral-content: oklch(90% 0.01 250); /* Semantic feedback colors */ - --color-info: oklch(80% 0.05 230); - --color-info-content: oklch(20% 0.05 240); - + --color-info: oklch(0.7 0.1 223); + --color-info-content: oklch(0.7 0.1 223); + --color-tip: oklch(0.26 0.10 113.29); + --color-tip-content: oklch(0.90 0.04 113.29); --color-success: oklch(72% 0.17 150); --color-success-content: oklch(15% 0.05 170); - --color-warning: oklch(75% 0.12 70); --color-warning-content: oklch(18% 0.07 50); - --color-error: oklch(66% 0.18 25); --color-error-content: oklch(15% 0.08 20); @@ -238,7 +239,7 @@ } .content table, -.content .highlight > pre, +.content .highlight>pre, .content pre.example { max-height: 70vh; margin: 1em 0; @@ -248,3 +249,100 @@ font-family: monospace, monospace; border: 1px dashed rgba(250, 100, 50, 0.5); } + +/* Admonition styles */ +aside.admonition { + --admonition-color: var(--color-neutral); + border-left-width: 3px; + border-left-style: solid; + border-color: var(--admonition-color); + padding: 1rem; + margin: 1.5rem 0; + border-radius: 0.25rem; + background-color: var(--color-base-200); +} + +aside.admonition .admonition-title { + margin: 0 0 0.75rem 0; + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 1rem; + font-weight: bold; + text-transform: capitalize; + color: var(--admonition-color); +} + +aside.admonition .admonition-title:before { + display: inline-block; + height: 1rem; + width: 1rem; + flex-shrink: 0; + overflow: visible; + background-color: var(--admonition-color); + vertical-align: middle; + content: ''; + mask-size: contain; + mask-position: center; + mask-repeat: no-repeat; +} + +aside.admonition .admonition-content> :last-child { + margin-bottom: 0; +} + +aside.admonition .admonition-content> :first-child { + margin-top: 0; +} + +/* Note admonition */ +aside.admonition[data-admonition-type="note"] { + --admonition-color: var(--color-info-content); + background-color: oklch(from var(--color-info) l c h / 0.1); +} + +aside.admonition[data-admonition-type="note"] .admonition-title::before { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='M12 16v-4'/%3E%3Cpath d='M12 8h.01'/%3E%3C/svg%3E"); +} + +/* Tip admonition */ +aside.admonition[data-admonition-type="tip"] { + --admonition-color: var(--color-tip-content); + background-color: oklch(from var(--color-tip) l c h / 0.1); +} + +aside.admonition[data-admonition-type="tip"] .admonition-title::before { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M9 11H6l3-9 2 9'/%3E%3Cpath d='M13 11h3l-3 9-2-9'/%3E%3C/svg%3E"); +} + +/* Important admonition */ +aside.admonition[data-admonition-type="important"] { + --admonition-color: var(--color-primary); + background-color: oklch(from var(--color-primary) l c h / 0.1); +} + +aside.admonition[data-admonition-type="important"] .admonition-title::before { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='M12 8v4'/%3E%3Cpath d='M12 16h.01'/%3E%3C/svg%3E"); +} + +/* Caution admonition */ +aside.admonition[data-admonition-type="caution"] { + --admonition-color: var(--color-warning); + background-color: oklch(from var(--color-warning) l c h / 0.1); +} + +aside.admonition[data-admonition-type="caution"] .admonition-title::before { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M10.29 3.86L1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0z'/%3E%3Cline x1='12' y1='9' x2='12' y2='13'/%3E%3Cline x1='12' y1='17' x2='12.01' y2='17'/%3E%3C/svg%3E"); +} + +/* Danger/Warning admonition */ +aside.admonition[data-admonition-type="danger"], +aside.admonition[data-admonition-type="warning"] { + --admonition-color: var(--color-error); + background-color: oklch(from var(--color-error) l c h / 0.1); +} + +aside.admonition[data-admonition-type="danger"] .admonition-title::before, +aside.admonition[data-admonition-type="warning"] .admonition-title::before { + mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolygon points='7.86 2 16.14 2 22 7.86 22 16.14 16.14 22 7.86 22 2 16.14 2 7.86 7.86 2'/%3E%3Cline x1='15' y1='9' x2='9' y2='15'/%3E%3Cline x1='9' y1='9' x2='15' y2='15'/%3E%3C/svg%3E"); +} \ No newline at end of file From f7b68abd1fdc5627bbfc93ef2ee63fcaa8d9804d Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Fri, 10 Oct 2025 10:43:58 +0200 Subject: [PATCH 3/7] feat: more progress on the article --- .../reading-writing-files-nodejs/index.md | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md index ac0daed..f9e1ada 100644 --- a/src/content/blog/reading-writing-files-nodejs/index.md +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -176,7 +176,17 @@ async function createBeepWav() { await createBeepWav() ``` -When working with binary data like this, we use the Node.js [Buffer](https://nodejs.org/api/buffer.html) object to organize and manipulate the binary data. A Buffer is a fixed-size sequence of bytes that provides a way to work with binary data directly. It's similar to an array of integers, but specifically designed for handling raw binary data efficiently. In our WAV example, we use `Buffer.alloc()` to create a buffer of the required size, then use methods like `writeUInt32LE()` and `writeInt16LE()` to write specific data types at specific positions in little-endian format. If you are curious to find out more about the binary structure of a WAV file, you can check out this excellent page on [Wikipedia](https://en.wikipedia.org/wiki/WAV). +This example demonstrates several key concepts for binary file manipulation in Node.js. The `encodeWavPcm16()` function creates a complete WAV file structure by constructing both the header and audio data sections. The WAV header contains metadata like file size, audio format, sample rate, and number of channels, while the data section contains the actual audio samples. + +The `generateBeepTone()` function creates audio samples using a sine wave mathematical formula, generating a pure tone at the specified frequency. Each sample represents the amplitude of the sound wave at a specific point in time, and when played back at the correct sample rate, these digital values recreate the original analog sound. + +Don't worry too much about the specifics of this example dealing with the WAV binary format - what matters here is learning that we can put arbitrary binary data into a buffer and write it into a file using `writeFile()`. The key takeaway is that Node.js treats all file operations the same way, whether you're writing text, JSON, images, audio, or any other type of data. + +When working with binary data like this, we use the Node.js [Buffer](https://nodejs.org/api/buffer.html) object to organize and manipulate the binary data. A Buffer is a fixed-size sequence of bytes that provides a way to work with binary data directly. It's similar to an array of integers, but specifically designed for handling raw binary data efficiently. In our WAV example, we use `Buffer.alloc()` to create a buffer of the required size, then use methods like `writeUInt32LE()` and `writeInt16LE()` to write specific data types at specific positions in little-endian format. + +:::note[WAV Binary File Structure] +If you are curious to find out more about the binary structure of a WAV file, you can check out this excellent page on [Wikipedia](https://en.wikipedia.org/wiki/WAV). +::: #### Reading Binary Data - Parse WAV file header @@ -255,6 +265,12 @@ if (!filePath) { await getWavDuration(filePath) ``` +This example showcases the reverse process - reading and parsing binary data from an existing WAV file. The `decodeWavHeader()` function demonstrates how to extract specific information from binary data by reading bytes at predetermined positions within the file structure. + +The function uses Buffer methods like `readUInt32LE()` and `readUInt16LE()` to interpret raw bytes as specific data types. For example, bytes 22-23 contain the number of channels, bytes 24-27 contain the sample rate, and bytes 40-43 contain the size of the audio data. By knowing the WAV file format specification, we can locate and extract these values. + +The duration calculation combines several pieces of metadata: we divide the total audio data size by the byte rate (bytes per second) to get the duration in seconds, then multiply by 1000 to convert to milliseconds. This demonstrates how understanding both the file format and basic math allows us to derive meaningful information from binary data. + You can try out this example with the beep WAV files we created in the previous example or, if you need some longer free WAV files, you can check out a website with free audio samples like [Zapsplat](https://www.zapsplat.com/). :::note[Binary file complexity] @@ -306,7 +322,7 @@ This concurrent approach provides a significant performance improvement. If we u #### Writing Multiple Files Concurrently -```javascript +```javascript {26-29} // write-multiple-files.js import { writeFile } from 'node:fs/promises' @@ -335,8 +351,8 @@ async function generateReports(data) { const promises = reports.map(({ filename, content }) => writeFile(filename, content, 'utf8'), ) - await Promise.all(promises) + console.log(`Successfully generated ${reports.length} report files`) } catch (error) { console.error('Failed to write report files:', error.message) @@ -486,7 +502,7 @@ Notice that when using `readFileSync()` and `writeFileSync()` we are not using ` However, if you care about extreme performance, there are specific cases where you might prefer the sync methods. Sync methods can occasionally be faster because they don't need to give back control to the event loop and, for example in the case of reading a file, the content of the file is immediately available once the filesystem operation completes without having to wait for the event loop to capture the event and give back control to our code. One case that comes to mind is when writing **non-concurrent CLI apps or scripts** that need to read or write a file before continuing with the next operation. -:::danger[Never Use Sync Methods in Concurrent Environments] +:::warning[Never Use Sync Methods in Concurrent Environments] **Do not use sync methods in concurrent environments such as web servers.** While you read or write a file synchronously, you will be blocking the event loop, which means that no other user requests will be processed during that time. This can cause noticeable delays for users - if a file operation takes 100ms, every user trying to access your server during that time will experience a 100ms delay. In a web application, this is unacceptable. @@ -502,7 +518,7 @@ This can cause noticeable delays for users - if a file operation takes 100ms, ev - Web servers (Express, Fastify, etc.) - Real-time applications - Any concurrent environment -::: + ::: ## Working with Large Files: Memory Considerations From c985d5b2f92e3ef6a9bfe1684035bed474889211 Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Fri, 10 Oct 2025 13:41:22 +0200 Subject: [PATCH 4/7] feat: more progress on the article --- .../reading-writing-files-nodejs/index.md | 95 +++++++++++++------ 1 file changed, 67 insertions(+), 28 deletions(-) diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md index f9e1ada..614d89c 100644 --- a/src/content/blog/reading-writing-files-nodejs/index.md +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -16,11 +16,17 @@ In this comprehensive guide, we'll focus on the modern approaches to file handli Let's dive in and master file operations in Node.js! +:::note[Code examples] +All the code examples in this article can be found on [GitHub](https://github.com/lmammino/reading-and-writing-files-with-nodejs-examples). +::: + ## Reading and Writing Files with Node.js Promises (fs/promises) -The most straightforward way to work with files in modern Node.js is using the `fs/promises` module with functions like `readFile()` and `writeFile()`. This gives us a clean, promise-based API that works beautifully with `async/await`. +The most straightforward way to work with files in modern Node.js is using the `node:fs/promises` module with functions like `readFile()` and `writeFile()`. This gives us a clean, promise-based API that works beautifully with `async/await`. + +Both `readFile()` and `writeFile()` return promises, and when we use `await`, we give control back to the event loop while the asynchronous file operation completes. When the operation finishes successfully, the event loop gives back control to our code and the execution moves to the next line of code. If the operation fails, the promise rejects and throws an error (which we can handle with a `try/catch` block). This non-blocking behavior is what makes Node.js so efficient at handling I/O operations in highly concurrent environments. -Both `readFile()` and `writeFile()` return promises, and when we use `await`, we give control back to the event loop while the asynchronous file operation completes. When the operation finishes successfully, execution moves to the next line of code. If the operation fails, the promise rejects and throws an error (which is why we need `try/catch` blocks). This non-blocking behavior is what makes Node.js so efficient at handling I/O operations in highly concurrent environments. +But enough theory, let's see this in action! ### Reading Files with fs/promises @@ -41,7 +47,7 @@ try { The `readFile()` function loads the entire file content into memory and returns it as a string (when you specify an encoding like 'utf8') or as a Buffer (when no encoding is specified). -:::tip[Why error handling matters] +:::tip[Handling errors when reading files] Notice how we wrapped our file operation in a `try/catch` block? File operations can fail for many reasons: - **ENOENT**: File or directory doesn't exist @@ -55,18 +61,17 @@ Always handle these errors gracefully to prevent your application from crashing. ### Writing Files with fs/promises -Writing files is just as straightforward. This example takes a JavaScript object, converts it to JSON, and saves it to a file: +Now that we've mastered reading files, what about creating them? Writing files is just as straightforward. This example takes a JavaScript object, converts it to JSON, and saves it to a file: -```javascript {10} +```javascript {9} // write-promises.js import { writeFile } from 'node:fs/promises' try { - const jsonData = JSON.stringify( - { name: 'John Doe', email: 'john@example.com' }, - null, - 2, - ) + const jsonData = JSON.stringify({ + name: 'John Doe', + email: 'john@example.com', + }) await writeFile('user-data.json', jsonData, 'utf8') console.log('User data saved successfully!') } catch (error) { @@ -75,6 +80,9 @@ try { } ``` +This example demonstrates the fundamental `writeFile()` operation: we're taking all our data (in this case, a JSON string) and flushing it to a file in a single atomic operation. The `writeFile()` function creates the file if it doesn't exist, or completely overwrites it if it does. The entire string content is written to disk at once, making this approach perfect for situations where you have all your data ready and want to persist it quickly. + +:::tip[Handling errors when writing files] Notice we're using `try/catch` here as well, because file write operations can fail for various reasons: - **EACCES**: Permission denied (can't write to the directory) @@ -82,10 +90,11 @@ Notice we're using `try/catch` here as well, because file write operations can f - **EROFS**: Read-only file system - **EISDIR**: Trying to write to a directory instead of a file - **ENOTDIR**: Part of the path is not a directory + ::: ### Reading and Writing Binary Files -Not all files are text-based. Here's a more advanced example showing how to work with binary data by creating and reading WAV audio files: +So far, we've been working with text files, but what happens when you need to handle images, videos, or audio files? Not all files are text-based, and Node.js handles binary data just as elegantly. Here's a more advanced example showing how to work with binary data by creating and reading WAV audio files: #### Writing Binary Data - Generate a WAV file @@ -180,7 +189,7 @@ This example demonstrates several key concepts for binary file manipulation in N The `generateBeepTone()` function creates audio samples using a sine wave mathematical formula, generating a pure tone at the specified frequency. Each sample represents the amplitude of the sound wave at a specific point in time, and when played back at the correct sample rate, these digital values recreate the original analog sound. -Don't worry too much about the specifics of this example dealing with the WAV binary format - what matters here is learning that we can put arbitrary binary data into a buffer and write it into a file using `writeFile()`. The key takeaway is that Node.js treats all file operations the same way, whether you're writing text, JSON, images, audio, or any other type of data. +Don't worry too much about the specifics of this example dealing with the WAV binary format! What matters here is learning that we can put arbitrary binary data into a buffer and write it into a file using `writeFile()`. The key takeaway is that Node.js treats all file operations the same way, whether you're writing text, JSON, images, audio, or any other type of data. When working with binary data like this, we use the Node.js [Buffer](https://nodejs.org/api/buffer.html) object to organize and manipulate the binary data. A Buffer is a fixed-size sequence of bytes that provides a way to work with binary data directly. It's similar to an array of integers, but specifically designed for handling raw binary data efficiently. In our WAV example, we use `Buffer.alloc()` to create a buffer of the required size, then use methods like `writeUInt32LE()` and `writeInt16LE()` to write specific data types at specific positions in little-endian format. @@ -271,7 +280,9 @@ The function uses Buffer methods like `readUInt32LE()` and `readUInt16LE()` to i The duration calculation combines several pieces of metadata: we divide the total audio data size by the byte rate (bytes per second) to get the duration in seconds, then multiply by 1000 to convert to milliseconds. This demonstrates how understanding both the file format and basic math allows us to derive meaningful information from binary data. +:::note[Finding free WAV files] You can try out this example with the beep WAV files we created in the previous example or, if you need some longer free WAV files, you can check out a website with free audio samples like [Zapsplat](https://www.zapsplat.com/). +::: :::note[Binary file complexity] This is a very simple example showing basic binary manipulation using only Node.js built-ins. We can handle the WAV format manually here because we're creating a minimal, single-channel PCM file with known parameters. @@ -281,7 +292,7 @@ For real-world audio applications, you'd typically want to use a comprehensive l ### Concurrent File Operations with Promises -One of the great advantages of promise-based file operations is the ability to read and write multiple files concurrently. Here's how you can do it: +Reading and writing single files is useful, but what if you need to process multiple files? Should you handle them one by one, or is there a faster way? One of the great advantages of promise-based file operations is the ability to read and write multiple files concurrently. Here's how you can do it: #### Reading Multiple Files Concurrently @@ -300,7 +311,7 @@ try { // Read all files concurrently and parse JSON // Note that we are not using await here, so the reads happen concurrently const promises = configFiles.map((file) => - readFile(file, 'utf8').then((content) => JSON.parse(content)), + readFile(file, 'utf8').then(JSON.parse), ) // Here we are using await, so we wait for all reads to complete before proceeding @@ -318,7 +329,13 @@ try { } ``` +This example demonstrates the power of promise chaining combined with concurrent operations. We start by mapping over an array of file names to create an array of promises. Each promise represents the complete operation of reading a file and parsing its JSON content. + +The key insight here is how we use `.then(JSON.parse)` — which is a shorthand for `.then(content => JSON.parse(content))` — to chain operations. First, `readFile(filename, 'utf8')` returns a promise that resolves to the raw file content as a string. Then, we chain `.then(JSON.parse)` to that promise, which creates a new promise that will resolve to the file content parsed from JSON into a JavaScript value (likely an object). In other words, this chaining transforms our original "read file" promise into a "read and parse JSON file" promise. + +:::important[Concurrent vs Sequential Operations] This concurrent approach provides a significant performance improvement. If we used multiple `await` statements (one for each file read), we would be processing the files sequentially. If each file read takes 10ms, we wouldn't be completing the operation in less than 40ms. With concurrent reads - by starting all promises first and then awaiting all of them with `Promise.all()` - we'll most likely be able to read all four files in around 10ms (roughly the time of the slowest individual read). +::: #### Writing Multiple Files Concurrently @@ -371,6 +388,14 @@ const reportData = { await generateReports(reportData) ``` +This example mirrors the concurrent reading pattern but for writing operations. We create an array of report objects, each containing a filename and content, then use `.map()` to transform this into an array of `writeFile()` promises. The key pattern here is the same: we start all write operations simultaneously (when we create the promises array) and then use `Promise.all()` to wait for all of them to complete. + +:::tip[JavaScript Promises are Eager] +It's important to remember that JavaScript promises are eager: once they are created, the underlying computation starts immediately. In this case, this means that once we call `writeFile()`, it will create a promise and start to write into the file immediately. We don't need to `await` the promise for the file operation to begin; the `await` only determines when our code waits for the operation to complete. +::: + +This approach is particularly useful for generating multiple related files, such as reports, logs, or configuration files. All files are written concurrently, which significantly improves performance compared to writing them one by one. If any write operation fails, the entire batch fails, which is often the desired behavior for related files that should be created as a unit. + :::tip[Promise.all() vs Promise.allSettled() - Choose the Right Tool] In these examples, we use `Promise.all()` because if any config file fails to load or any report fails to write, we can't continue - all operations are required for the application to function properly. @@ -386,7 +411,7 @@ Learn more: [Promise.all() on MDN](https://developer.mozilla.org/en-US/docs/Web/ ### Working with Directories -You'll often need to work with directories too: +Files don't exist in isolation, they live in directories. What if you need to process all files in a folder, or create directories dynamically? You'll often need to work with directories too: ```javascript {8,11,14,15,17} // process-directory.js @@ -468,7 +493,7 @@ This approach is much more reliable than using relative paths like `'./config.js ### Async vs Sync File Operations: When to Use Which -Before diving into more advanced file handling techniques, let's compare the promise-based approaches we've seen with the synchronous alternatives Node.js provides. +We've been using async operations throughout this guide, but you might be wondering: "Are there simpler, synchronous alternatives?" The answer is yes, but when should you use them? Before diving into more advanced file handling techniques, let's compare the promise-based approaches we've seen with the synchronous alternatives Node.js provides. Node.js also offers synchronous versions of file operations like `readFileSync()` and `writeFileSync()`: @@ -496,11 +521,15 @@ try { } ``` -Notice that when using `readFileSync()` and `writeFileSync()` we are not using `await`. This is because these methods are synchronous: they block the event loop until the operation is completed, which means no other JavaScript code is executed while the file read/write operation is in progress. +Notice that when using `readFileSync()` and `writeFileSync()` we are not using `await`. This is because these functions are synchronous: they block the event loop until the operation is completed, which means no other JavaScript code is executed while the file read/write operation is in progress. **My recommendation: I generally prefer to avoid synchronous file functions entirely and always go for the async approach for consistency.** This keeps your codebase uniform and prevents accidentally blocking the event loop in unexpected places. This makes even more sense since for a few years Node.js has added support for **top-level await** (being able to use `await` outside an async function), so using asynchronous file operations is just as easy as using synchronous ones. -However, if you care about extreme performance, there are specific cases where you might prefer the sync methods. Sync methods can occasionally be faster because they don't need to give back control to the event loop and, for example in the case of reading a file, the content of the file is immediately available once the filesystem operation completes without having to wait for the event loop to capture the event and give back control to our code. One case that comes to mind is when writing **non-concurrent CLI apps or scripts** that need to read or write a file before continuing with the next operation. +However, if you care about extreme performance, there are specific cases where you might prefer the sync methods. Sync methods can occasionally be slightly faster because they don't need to give back control to the event loop and, for example in the case of reading a file, the content of the file is immediately available once the filesystem operation completes without having to wait for the event loop to capture the event and give back control to our code. One case that comes to mind is when writing **non-concurrent CLI apps or scripts** that need to read or write a file before continuing with the next operation. + +:::note[Always run your benchmarks!] +If you are wondering how much faster sync methods are compared to async ones, the answer is: _it depends!_ The only way to know for sure is to run benchmarks in your specific environment and use case. Factors like filesystem speed, file size, and system load can all impact performance. With that being said, we do expect the difference to be negligible in most cases, which is one more reason to prefer async methods for consistency. +::: :::warning[Never Use Sync Methods in Concurrent Environments] **Do not use sync methods in concurrent environments such as web servers.** While you read or write a file synchronously, you will be blocking the event loop, which means that no other user requests will be processed during that time. @@ -520,9 +549,9 @@ This can cause noticeable delays for users - if a file operation takes 100ms, ev - Any concurrent environment ::: -## Working with Large Files: Memory Considerations +## Working with Large Files -The promise-based approach we've seen so far is perfect for small to medium-sized files. However, there's a significant limitation: **everything gets loaded into memory at once**. +The promise-based approaches we've covered work brilliantly for everyday files - but what happens when you need to process a massive CSV file, or a multi-gigabyte log file? The promise-based approach we've seen so far is perfect for small to medium-sized files. However, there's a significant limitation: **everything gets loaded into memory at once**. Imagine you're trying to read a 2GB log file using `readFile()`. Your Node.js process will attempt to load all 2GB into memory simultaneously. This can lead to several problems: @@ -560,7 +589,7 @@ try { } ``` -So does this mean that Node.js can't handle big files?! Of course not, we just need to use different tools to do that! The trick is to make sure we don't load ALL the data into memory in one go, but we process the data in smaller incremental chunks! +So does this mean that Node.js can't handle big files?! Of course not, Node.js is actually quite good at handling them... we just need to use different tools to do that! The trick is to make sure we don't load ALL the data into memory in one go, but we process the data in smaller incremental chunks! ### When to Use Promise-Based Methods @@ -573,7 +602,7 @@ Promise-based file operations are great when: ## Advanced Node.js File Operations with File Handles -When you need more control over file operations, Node.js provides lower-level APIs using file handles. This approach allows you to read and write files incrementally, giving you fine-grained control over memory usage. +So how do we handle those massive files without running out of memory? When you need more control over file operations, Node.js provides lower-level APIs using file handles. This approach allows you to read and write files incrementally, giving you fine-grained control over memory usage. If you have ever read data from a file or written data into a file in a low-level language such as C, this approach will seem familiar. @@ -632,11 +661,15 @@ This example demonstrates the core concepts of working with file handles in Node The `read()` method allows us to read a specific number of bytes from a specific position in the file. In our loop, we read 1KB chunks at a time, processing each chunk before moving to the next. This approach keeps memory usage low and predictable, regardless of the file size. -Most importantly, we need to make sure we clean up resources, which is why we use a `finally` block. This way, the cleanup code (calling `fileHandle.close()`) is executed whether everything goes well or if there's an error. This allows us to retain a clean state and lets Node.js reclaim resources that aren't needed anymore. Failing to close file handles can lead to resource leaks and eventually cause your application to run out of available file descriptors. +Most importantly, we need to make sure we clean up resources, which is why we use a `finally` block. This way, the cleanup code (calling `fileHandle.close()`) is executed whether everything goes well or if there's an error. This allows us to retain a clean state and lets Node.js reclaim resources that aren't needed anymore. + +:::warning[Always Close File Handles] +Failing to close file handles can lead to resource leaks and eventually cause your application to run out of available file descriptors. +::: ### Writing Files Incrementally -Let's say we want to create a file that contains 1 million unique voucher codes. If we prepopulate all the data in memory, that would require a significant amount of memory. Instead, we can generate vouchers in small chunks and append them to a file as we go: +Suppose we need a file with one million unique voucher codes for a later print run. Preloading them all in memory would be wasteful, so we stream the work: generate codes in small batches and append each batch to the file. ```javascript {20,42,56} // generate-voucher-codes.js @@ -720,14 +753,14 @@ While file handles give you precise control, they require you to manage a lot of - **Error handling** - More complex error scenarios to handle - **Resource cleanup** - You must remember to close file handles -This low-level approach is powerful but verbose. For most use cases, there's a better solution: **streams**. +This low-level approach is powerful but verbose. For most use cases, there's a better solution: **Node.js streams**. ## Node.js Streams: Memory-Efficient File Processing -Streams provide the memory efficiency of low-level file operations with a much more ergonomic API. They're perfect for processing large files without loading everything into memory. +File handles give us control, but they require a lot of manual work. Is there a middle ground that gives us memory efficiency without all the complexity? Absolutely! Streams provide the memory efficiency of low-level file operations with a much more ergonomic API. They're perfect for processing large files without loading everything into memory. -:::tip[Want to Master Node.js Streams?] -We're giving away for free an entire chapter from our book "Node.js Design Patterns" dedicated to learning Streams! +:::tip[Want to Master Node.js Streams (for FREE)?] +We're giving away for free an entire chapter from our book ["Node.js Design Patterns"](/) entirely dedicated to learning Streams! **Chapter 6: Coding with Streams** - 80 pages packed with practical examples, real-world insights, and powerful design patterns to help you write faster, leaner, and more scalable Node.js code. @@ -1086,6 +1119,8 @@ await analyzeLogFile('./logs/access.log') ### Best Practices Summary +We've covered a lot of ground in this guide - from simple file reading to advanced streaming patterns. But with so many options available, how do you choose the right approach for your specific use case? Here are the key principles to guide your decisions: + 1. **Choose the right approach for your use case**: - **Small files (< 100MB)**: Use `fs/promises` with `readFile()` and `writeFile()` - **Large files or unknown sizes**: Use streams with `createReadStream()` and `createWriteStream()` @@ -1210,6 +1245,8 @@ Remember that file operations are often I/O bound, so proper error handling and ## Frequently Asked Questions +Still have questions about Node.js file operations? You're not alone! Here are the most common questions developers ask when mastering file handling in Node.js: + ### What's the difference between sync and async file operations in Node.js? Synchronous file operations like `readFileSync()` block the event loop until the operation completes, meaning no other JavaScript code can execute during that time. Asynchronous operations like `readFile()` don't block the event loop, allowing other code to run while the file operation happens in the background. Always use async operations in web servers and concurrent applications to avoid blocking other requests. @@ -1238,6 +1275,8 @@ Use `Promise.all()` or `Promise.allSettled()` with an array of promises to proce ## Take Your Node.js Skills to the Next Level +Congratulations! You've just mastered one of the most fundamental aspects of Node.js development. But this is just the beginning of your journey to becoming a Node.js expert. + If you found value in this comprehensive guide to file operations, you'll love **Node.js Design Patterns** - the definitive resource designed to elevate Node.js developers from junior to senior level. This book dives deep into the patterns, techniques, and best practices that separate good Node.js code from great Node.js code. From fundamental concepts like the ones covered in this article to advanced architectural patterns for building scalable applications, it provides the knowledge you need to write professional, maintainable Node.js code. From bff23cd2b0e87f966924de66ad09a96fa5704443 Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Sun, 12 Oct 2025 20:23:20 +0200 Subject: [PATCH 5/7] chore: more progress on article --- ec.config.mjs | 3 + package.json | 1 + pnpm-lock.yaml | 10 + .../reading-writing-files-nodejs/index.md | 460 +++++++----------- 4 files changed, 178 insertions(+), 296 deletions(-) diff --git a/ec.config.mjs b/ec.config.mjs index df98474..2f5af80 100644 --- a/ec.config.mjs +++ b/ec.config.mjs @@ -1,8 +1,11 @@ +import { pluginCollapsibleSections } from '@expressive-code/plugin-collapsible-sections' + const config = { themes: ['material-theme-ocean', 'min-light'], themeCssSelector(theme) { return `[data-code-theme='${theme.name}']` }, + plugins: [pluginCollapsibleSections({})], useDarkModeMediaQuery: false, } diff --git a/package.json b/package.json index 5228749..a85c4bb 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@astrojs/partytown": "^2.1.4", "@astrojs/react": "^4.3.1", "@astrojs/sitemap": "^3.6.0", + "@expressive-code/plugin-collapsible-sections": "^0.41.3", "@lucide/astro": "^0.522.0", "@radix-ui/react-slot": "^1.2.3", "@tailwindcss/typography": "^0.5.16", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 977f0fd..28aed14 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@astrojs/sitemap': specifier: ^3.6.0 version: 3.6.0 + '@expressive-code/plugin-collapsible-sections': + specifier: ^0.41.3 + version: 0.41.3 '@lucide/astro': specifier: ^0.522.0 version: 0.522.0(astro@5.13.10(@types/node@22.15.29)(jiti@2.4.2)(lightningcss@1.30.1)(rollup@4.41.1)(typescript@5.8.3)) @@ -497,6 +500,9 @@ packages: '@expressive-code/core@0.41.3': resolution: {integrity: sha512-9qzohqU7O0+JwMEEgQhnBPOw5DtsQRBXhW++5fvEywsuX44vCGGof1SL5OvPElvNgaWZ4pFZAFSlkNOkGyLwSQ==} + '@expressive-code/plugin-collapsible-sections@0.41.3': + resolution: {integrity: sha512-cuHIN7Ipl7gUcaWFfsgy6G3wn0Svk8dQ6WKXNQha63BURbm7CSBhD6y9qFGeIOrxaJtvH4Pj3Xb4C2Ni0OVwYA==} + '@expressive-code/plugin-frames@0.41.3': resolution: {integrity: sha512-rFQtmf/3N2CK3Cq/uERweMTYZnBu+CwxBdHuOftEmfA9iBE7gTVvwpbh82P9ZxkPLvc40UMhYt7uNuAZexycRQ==} @@ -3245,6 +3251,10 @@ snapshots: unist-util-visit: 5.0.0 unist-util-visit-parents: 6.0.1 + '@expressive-code/plugin-collapsible-sections@0.41.3': + dependencies: + '@expressive-code/core': 0.41.3 + '@expressive-code/plugin-frames@0.41.3': dependencies: '@expressive-code/core': 0.41.3 diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md index 614d89c..d443384 100644 --- a/src/content/blog/reading-writing-files-nodejs/index.md +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -733,7 +733,7 @@ async function generateVoucherFile(filePath, totalVouchers) { } // Usage - Generate 1 million voucher codes -await generateVoucherFile('voucher-codes.txt', 1000000) +await generateVoucherFile('voucher-codes.txt', 1_000_000) ``` This example shows how to efficiently generate and write large amounts of data without overwhelming memory. The `open()` function creates a file handle with the `'w'` flag, which opens the file for writing (creating it if it doesn't exist, or truncating it if it does). @@ -786,17 +786,15 @@ Streams offer several key benefits: ### Reading Files with Streams -Here's how to read a file using a readable stream: +Ready to see the magic of streams in action? Let's build a CLI utility that takes a text file of arbitrary size and counts the number of characters and lines in it, all while barely using any memory. -```javascript +```javascript {5-7,12,20,26} // process-large-file-stream.js import { createReadStream } from 'node:fs' -import { pipeline } from 'node:stream/promises' async function processLargeFile(filePath) { const readStream = createReadStream(filePath, { encoding: 'utf8', - highWaterMark: 16 * 1024, // 16KB chunks }) let lineCount = 0 @@ -825,301 +823,207 @@ async function processLargeFile(filePath) { await processLargeFile('huge-log-file.txt') ``` -### Writing Files with Streams +This example demonstrates the elegance of stream-based file processing. We create a readable stream with `createReadStream()` and set up event handlers to process the data as it flows through. -Writing with streams is equally straightforward: +The key advantage here is that we're processing the file incrementally - each chunk is handled as soon as it's read, without waiting for the entire file to load into memory. This makes it possible to process files of any size using a constant, predictable amount of memory. The stream emits three important events: `data` (when a chunk is available), `end` (when the entire file has been read), and `error` (if something goes wrong). -```javascript -// generate-large-file.js -import { createWriteStream } from 'node:fs' -import { pipeline } from 'node:stream/promises' -import { Readable } from 'node:stream' +This pattern is perfect for analyzing log files, processing CSV data, or any scenario where you need to examine file content without loading everything into memory at once. -async function generateLargeFile(filePath, numberOfLines) { - const writeStream = createWriteStream(filePath, { - encoding: 'utf8', - highWaterMark: 64 * 1024, // 64KB buffer - }) +:::tip[UTF-8 Encoding and Multibyte Character Handling] +By specifying `encoding: 'utf8'`, the stream takes care of processing multibyte UTF-8 characters for us automatically. Here's why this matters: chunking data into arbitrary windows of bytes (64KB by default) might happen to truncate a multibyte character between chunks. For example, a 2-byte character might have the first byte at the end of one chunk and the second byte at the beginning of the following chunk. - // Create a readable stream that generates data - const dataGenerator = new Readable({ - read() { - if (numberOfLines > 0) { - this.push(`Line ${numberOfLines}: This is some sample data\n`) - numberOfLines-- - } else { - this.push(null) // End the stream - } - }, - }) - - try { - // Use pipeline for proper error handling and cleanup - await pipeline(dataGenerator, writeStream) - console.log('File generation completed!') - } catch (error) { - console.error('Pipeline failed:', error.message) - throw error - } -} +This can be problematic because if you print chunks without stitching them together, you might end up with broken characters displayed between chunks. Thankfully, Node.js takes care of doing the stitching for us - it will move the incomplete multibyte character from one chunk to the following chunk, so every chunk is guaranteed to contain a valid UTF-8 string (assuming the source file contains valid UTF-8 text). +::: -// Usage: Generate a file with 1 million lines -await generateLargeFile('generated-data.txt', 1000000) -``` +### Writing Files with Streams -### Stream Composition and Processing +Now let's flip the script and see streams in action for writing! Remember our voucher code generator that could create millions of codes? Let's revisit that example using streams for a more ergonomic and manageable approach. We'll build the same voucher generator but with cleaner, simpler code that's easier to understand and maintain. -One of the most powerful features of streams is the ability to compose them. Here's an example that reads a CSV file, processes it, and writes the results: +```javascript {17,23,45-48, 56} collapse={8-13} +// generate-voucher-codes-stream.js +import { once } from 'node:events' +import { createWriteStream } from 'node:fs' -```javascript -// csv-processor.js -import { createReadStream, createWriteStream } from 'node:fs' -import { pipeline } from 'node:stream/promises' -import { Transform } from 'node:stream' - -// Custom transform stream to process CSV data -class CSVProcessor extends Transform { - constructor(options = {}) { - super({ objectMode: true, ...options }) - this.headers = null - this.lineCount = 0 +// This is for demonstration purposes only. +// Ideally, voucher codes should be generated using a secure random generator +function generateVoucherCode() { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + let result = '' + for (let i = 0; i < 8; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)) } + return result +} - _transform(chunk, encoding, callback) { - const lines = chunk.toString().split('\n') +async function generateVoucherFile(filePath, totalVouchers) { + const writeStream = createWriteStream(filePath) - for (const line of lines) { - if (!line.trim()) continue + const chunkSize = 1000 // Generate 1000 vouchers per chunk + let vouchersGenerated = 0 + let hasError = false - if (!this.headers) { - this.headers = line.split(',') - continue - } + writeStream.on('error', (err) => { + console.error('Error writing to file:', err) + hasError = true + }) - const values = line.split(',') - const record = {} + while (vouchersGenerated < totalVouchers && !hasError) { + // Generate a chunk of voucher codes + const vouchersInThisChunk = Math.min( + chunkSize, + totalVouchers - vouchersGenerated, + ) + const vouchers = [] - this.headers.forEach((header, index) => { - record[header.trim()] = values[index]?.trim() || '' - }) + for (let i = 0; i < vouchersInThisChunk; i++) { + vouchers.push(generateVoucherCode()) + } - // Transform the record (e.g., uppercase all names) - if (record.name) { - record.name = record.name.toUpperCase() - } + // Convert to buffer and write to file + const chunk = `${vouchers.join('\n')}\n` + const buffer = Buffer.from(chunk, 'utf8') - this.push(JSON.stringify(record) + '\n') - this.lineCount++ + // Write and handle backpressure + const canContinue = writeStream.write(buffer) + if (!canContinue) { + await once(writeStream, 'drain') } - callback() - } + vouchersGenerated += vouchersInThisChunk - _flush(callback) { - console.log(`Processed ${this.lineCount} records`) - callback() + console.log(`Generated ${vouchersGenerated}/${totalVouchers} vouchers`) } -} - -async function processCSVFile(inputPath, outputPath) { - try { - await pipeline( - createReadStream(inputPath, { encoding: 'utf8' }), - new CSVProcessor(), - createWriteStream(outputPath, { encoding: 'utf8' }), - ) - console.log('CSV processing completed!') - } catch (error) { - console.error('CSV processing failed:', error.message) - throw error - } + console.log(`Successfully generated ${totalVouchers} voucher codes!`) + writeStream.end() } -// Usage -await processCSVFile('users.csv', 'processed-users.json') +// Usage - Generate 1 million voucher codes +await generateVoucherFile('voucher-codes.txt', 1_000_000) ``` -### Handling Backpressure +This stream-based approach transforms our file generation process into a more ergonomic operation. The `createWriteStream()` function creates a writable stream that we can feed data to incrementally. Unlike our earlier file handle approach, we don't need to manually track file positions - the stream automatically advances its internal cursor for us after each write. -Streams automatically handle backpressure - the situation where data is being produced faster than it can be consumed. Here's how you can monitor and control it: +Notice how we handle errors differently here - instead of using try/catch blocks, we set up an error event handler that sets the `hasError` flag. This event-driven approach lets the stream manage error propagation naturally. -```javascript -// backpressure-handling.js -import { createReadStream, createWriteStream } from 'node:fs' - -function copyFileWithBackpressureHandling(source, destination) { - return new Promise((resolve, reject) => { - const readStream = createReadStream(source) - const writeStream = createWriteStream(destination) - - readStream.on('data', (chunk) => { - const canContinue = writeStream.write(chunk) - - if (!canContinue) { - // Backpressure detected - pause reading - console.log('Backpressure detected, pausing read stream') - readStream.pause() - - // Resume when drain event is emitted - writeStream.once('drain', () => { - console.log('Resuming read stream') - readStream.resume() - }) - } - }) +The `canContinue` variable captures an important concept: **backpressure handling**. When `writeStream.write()` returns `false`, it means the internal buffer is full and we should pause until the 'drain' event fires. This prevents memory buildup when we're generating data faster than it can be written to disk. - readStream.on('end', () => { - writeStream.end() - }) +Compare this to our file handle version: we've eliminated manual position tracking, simplified the control flow, and gained automatic buffering. While this example could still be greatly simplified using higher-level stream utilities, it's already much simpler than using file handles directly. The stream manages the underlying file operations, letting us focus on generating data. - writeStream.on('finish', () => { - console.log('File copy completed!') - resolve() - }) +### Stream Composition and Processing - readStream.on('error', reject) - writeStream.on('error', reject) - }) +Now let's see the true power of streams by creating a much more elegant version of our voucher generator. One of the most powerful features of streams is their composability - you can combine different streams to create sophisticated data processing pipelines: + +```javascript {17-27,30,31,34} collapse={9-14} +// stream-composition.js +import { createWriteStream } from 'node:fs' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' + +// This is for demonstration purposes only. +// Ideally, voucher codes should be generated using a secure random generator +function generateVoucherCode() { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + let result = '' + for (let i = 0; i < 8; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)) + } + return result } -// Usage -await copyFileWithBackpressureHandling('large-video.mp4', 'copy-video.mp4') -``` +class VoucherGeneratorStream extends Readable { + constructor(options) { + super({ ...options, objectMode: true }) + } -### When to Use Streams + _read(_size) { + const chunk = `${generateVoucherCode()}\n` + this.push(chunk) + } +} -Streams are the best choice when: +const totalVouchers = 1_000_000 +const sourceStream = new VoucherGeneratorStream().take(totalVouchers) // Generate 1 million vouchers +const destinationStream = createWriteStream('voucher-codes.txt') -- **Processing large files** (anything over 100MB) -- **Real-time data processing** (logs, live data feeds) -- **Memory is limited** (cloud functions, containers) -- **You need composability** (multiple processing steps) -- **Handling unknown file sizes** (user uploads, network data) +try { + await pipeline(sourceStream, destinationStream) + console.log(`Successfully generated ${totalVouchers} voucher codes!`) +} catch (err) { + console.error('Pipeline failed:', err) +} +``` -## Real-World Node.js File Operations: Examples and Best Practices +This example is functionally equivalent to our previous voucher generator but offers a much more concise and readable implementation. Let's break down what makes this approach so powerful: -Let's look at some real-world scenarios and best practices for file operations. +**Custom Readable Stream**: We create a `VoucherGeneratorStream` class that extends Node.js's `Readable` stream. The `_read()` method is called whenever the stream needs more data - it generates one voucher code per call and pushes it to the stream. Notice we use `objectMode: true` in the constructor, which tells the stream to treat each voucher string as an individual chunk rather than trying to buffer multiple strings together. -### Example 1: Processing User Uploads +**Endless Stream with Limits**: Our `VoucherGeneratorStream` is essentially endless - it would keep generating voucher codes forever. That's where the `.take(totalVouchers)` utility function comes in. This stream method limits our endless stream to exactly 1 million vouchers, automatically ending the stream once that limit is reached. -```javascript -// process-upload.js -import { createWriteStream, createReadStream } from 'node:fs' -import { pipeline } from 'node:stream/promises' -import { Transform } from 'node:stream' -import crypto from 'node:crypto' - -class FileHasher extends Transform { - constructor() { - super() - this.hash = crypto.createHash('sha256') - this.size = 0 - } +**Automatic Pipeline Management**: The `pipeline()` function is where the magic happens. It connects our voucher generator stream to the file write stream, automatically handling data flow, backpressure, and error propagation. When the generator produces data faster than the file can be written, `pipeline()` automatically pauses the generator until the file stream catches up. No manual `canContinue` checks or `drain` event handling needed! - _transform(chunk, encoding, callback) { - this.hash.update(chunk) - this.size += chunk.length - this.push(chunk) // Pass chunk through unchanged - callback() - } +The beauty of this approach is its declarative nature - we describe _what_ we want (generate vouchers and write them to a file) rather than _how_ to manage the low-level details. The stream infrastructure handles buffering, backpressure, and coordination for us. - _flush(callback) { - this.digest = this.hash.digest('hex') - callback() - } -} +:::tip[Learn More About Streams] +There are many stream details we've glossed over here - transform streams, duplex streams, and advanced composition patterns. For a deep dive into Node.js streams, check out the [FREE streams chapter from our Node.js Design Patterns book](/#free-chapter). +::: -async function saveUserUpload(uploadStream, filename) { - const hasher = new FileHasher() - const writeStream = createWriteStream(`./uploads/${filename}`) +:::tip[Using pipeline() with Generators] +It's worth noting that the `pipeline()` function also supports **iterables** and **async iterables** (including **generator functions**), which can allow us to make the code even more concise. - try { - await pipeline(uploadStream, hasher, writeStream) +
+ Alternative implementation using an async Generator Function (click to expand) - console.log(`File saved: ${filename}`) - console.log(`Size: ${hasher.size} bytes`) - console.log(`SHA256: ${hasher.digest}`) +```javascript {5,14,22} +// stream-composition-gen.js +import { createWriteStream } from 'node:fs' +import { pipeline } from 'node:stream/promises' - return { - filename, - size: hasher.size, - hash: hasher.digest, +function* voucherCodesGen() { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + while (true) { + // This is for demonstration purposes only. + // Ideally, voucher codes should be generated using a secure random generator + let result = '' + for (let i = 0; i < 8; i++) { + result += chars.charAt(Math.floor(Math.random() * chars.length)) } - } catch (error) { - console.error('Upload failed:', error.message) - throw error + yield `${result}\n` } } -``` - -### Example 2: Log File Analysis - -```javascript -// analyze-logs.js -import { createReadStream } from 'node:fs' -import { createInterface } from 'node:readline' -async function analyzeLogFile(logPath) { - const fileStream = createReadStream(logPath) - const rl = createInterface({ - input: fileStream, - crlfDelay: Infinity, // Handle Windows line endings - }) +const totalVouchers = 1_000_000 // Generate 1 million vouchers +const destinationStream = createWriteStream('voucher-codes.txt') - const stats = { - totalLines: 0, - errorLines: 0, - warningLines: 0, - ipAddresses: new Set(), - statusCodes: new Map(), - } - - for await (const line of rl) { - stats.totalLines++ +try { + await pipeline(voucherCodesGen().take(totalVouchers), destinationStream) + console.log(`Successfully generated ${totalVouchers} voucher codes!`) +} catch (err) { + console.error('Pipeline failed:', err) +} +``` - // Simple log parsing (adjust for your log format) - if (line.includes('ERROR')) { - stats.errorLines++ - } else if (line.includes('WARN')) { - stats.warningLines++ - } +
- // Extract IP addresses (simple regex) - const ipMatch = line.match(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/) - if (ipMatch) { - stats.ipAddresses.add(ipMatch[0]) - } +If you want to learn more about JavaScript iteration protocols, including iterables, async iterables, and generator functions, check out our article dedicated to [JavaScript Async Iterators](/blog/javascript-async-iterators) where we cover these concepts in detail. - // Extract HTTP status codes - const statusMatch = line.match(/\s(\d{3})\s/) - if (statusMatch) { - const status = statusMatch[1] - stats.statusCodes.set(status, (stats.statusCodes.get(status) || 0) + 1) - } - } +::: - console.log('Log Analysis Results:') - console.log(`Total lines: ${stats.totalLines}`) - console.log(`Error lines: ${stats.errorLines}`) - console.log(`Warning lines: ${stats.warningLines}`) - console.log(`Unique IP addresses: ${stats.ipAddresses.size}`) - console.log('Status code distribution:') +### When to Use Streams - for (const [code, count] of stats.statusCodes) { - console.log(` ${code}: ${count}`) - } +So, to summarize, when should you reach for streams? - return stats -} +Streams are the best choice when: -// Usage -await analyzeLogFile('./logs/access.log') -``` +- **Processing large files** (anything over 100MB) +- **Real-time data processing** (logs, live data feeds) +- **Memory is limited** (cloud functions, containers) +- **You need composability** (multiple processing steps) +- **Handling unknown file sizes** (user uploads, network data) -### Best Practices Summary +## Node.js File Operations: Best Practices Summary -We've covered a lot of ground in this guide - from simple file reading to advanced streaming patterns. But with so many options available, how do you choose the right approach for your specific use case? Here are the key principles to guide your decisions: +We've covered a comprehensive range of file operation techniques in Node.js, from simple promise-based methods to advanced streaming patterns. Here are the key principles and some extra tips to guide your decisions when choosing the right approach for your specific use case: 1. **Choose the right approach for your use case**: - **Small files (< 100MB)**: Use `fs/promises` with `readFile()` and `writeFile()` @@ -1133,7 +1037,9 @@ We've covered a lot of ground in this guide - from simple file reading to advanc - Use `Promise.allSettled()` when you can tolerate partial failures - Read/write multiple files concurrently when possible -3. **Always handle errors properly**: +3. **Handle specific errors**: + + It's good practice to handle specific error codes to provide meaningful feedback to the user or take specific corrective action: ```javascript // error-handling-example.js @@ -1154,16 +1060,20 @@ We've covered a lot of ground in this guide - from simple file reading to advanc 4. **Use appropriate buffer sizes for streams**: + Stream objects will load data in chunks into an internal buffer. The default chunk size is 64KB (or 16 objects if using object mode), which is a good balance for most use cases. However, you can adjust the `highWaterMark` option when creating streams to optimize performance based on your specific needs. When processing large amounts of data, if you can tolerate allocating more memory, increasing the chunk size can reduce the number of I/O operations and improve throughput. + ```javascript // stream-buffer-size.js // For large files, use bigger chunks const stream = createReadStream(path, { - highWaterMark: 64 * 1024, // 64KB chunks + highWaterMark: 128 * 1024, // 64KB chunks }) ``` 5. **Always clean up resources**: + When using file handles or streams, ensure you properly close them to avoid resource leaks. Use `finally` blocks or event handlers to guarantee cleanup even in the face of errors. + ```javascript // resource-cleanup.js // File handles @@ -1178,66 +1088,20 @@ We've covered a lot of ground in this guide - from simple file reading to advanc stream.on('error', () => stream.destroy()) ``` -6. **Use modern JavaScript patterns**: - - Array destructuring for concurrent operations - - Promise chaining for data transformation - - Proper error handling with `try/catch` blocks + When using `pipeline()`, it automatically handles cleanup for you, so prefer it when possible. -7. **Consider using pipeline() for stream operations**: ```javascript // pipeline-example.js // Better error handling and cleanup await pipeline(source, transform, destination) ``` -## Summary and Conclusion - -We've covered a comprehensive range of file operation techniques in Node.js, from simple promise-based methods to advanced streaming patterns. Here's a quick recap of when to use each approach: - -**Use `fs/promises` (async methods) when:** - -- Files are small to medium-sized (under 100MB) -- You need the entire file content at once -- Working in concurrent environments (web servers, APIs) -- You want consistency across your codebase -- Memory usage isn't a critical constraint - -**Use sync methods (`readFileSync`, `writeFileSync`) when:** - -- Building CLI tools or scripts -- Working in non-concurrent environments -- You need to complete file operations before continuing -- Extreme performance is critical for simple operations - -**Use concurrent operations when:** - -- Reading/writing multiple files -- Operations can be performed in parallel -- You want to maximize I/O efficiency - -**Use file handles when:** - -- You need precise control over read/write operations -- Working with specific file positions or ranges -- Building low-level file processing tools -- Performance optimization is critical - -**Use streams when:** - -- Processing large files (over 100MB) -- Memory efficiency is important -- You need to compose multiple operations -- Handling real-time or unknown-sized data -- Building scalable applications -- Working with binary data formats - -**Key principles for modern Node.js file operations:** +6. **Use modern JavaScript patterns**: + - Array destructuring for concurrent operations + - Promise chaining for data transformation + - Proper error handling with `try/catch` blocks -1. **Prefer async by default** - Use promise-based methods unless you have a specific reason to use sync methods -2. **Leverage concurrency** - Process multiple files simultaneously when possible -3. **Handle errors gracefully** - Always use proper error handling with specific error codes -4. **Choose the right tool** - Match your approach to your file size and use case -5. **Use modern JavaScript** - Take advantage of array destructuring, promise chaining, and async/await +## Summary and Conclusion The key to mastering file operations in Node.js is understanding these trade-offs and choosing the right approach for your specific use case. Start with the simple promise-based methods for most scenarios, leverage concurrency when processing multiple files, and move to streams when you need better performance and memory efficiency for large files. @@ -1269,7 +1133,7 @@ Yes! Modern Node.js supports top-level await in ES modules, so you can use `awai ### How do I process multiple files concurrently in Node.js? -Use `Promise.all()` or `Promise.allSettled()` with an array of promises to process multiple files simultaneously. For example: `await Promise.all(filenames.map(name => readFile(name)))`. This is much faster than processing files sequentially, especially for I/O-bound operations. +Use `Promise.all()` or `Promise.allSettled()` with an array of promises to process multiple files simultaneously. For example: `await Promise.all(filenames.map(name => readFile(name)))`. This is much faster than processing files sequentially, especially for I/O-bound operations. If you are processing large files, you might want to consider using streams. You can create multiple stream objects and pipeline and run them concurrently. --- @@ -1282,3 +1146,7 @@ If you found value in this comprehensive guide to file operations, you'll love * This book dives deep into the patterns, techniques, and best practices that separate good Node.js code from great Node.js code. From fundamental concepts like the ones covered in this article to advanced architectural patterns for building scalable applications, it provides the knowledge you need to write professional, maintainable Node.js code. **Ready to master Node.js?** Visit our [homepage](/) to discover how Node.js Design Patterns can accelerate your development journey and help you build better applications with confidence. + +``` + +``` From ebbc27df9b39c5d2c55fa50cb11414430509a454 Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Sun, 12 Oct 2025 23:32:46 +0200 Subject: [PATCH 6/7] feat: Completed article --- .../reading-writing-files-nodejs/index.md | 78 +++++++++---------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/src/content/blog/reading-writing-files-nodejs/index.md b/src/content/blog/reading-writing-files-nodejs/index.md index d443384..2941524 100644 --- a/src/content/blog/reading-writing-files-nodejs/index.md +++ b/src/content/blog/reading-writing-files-nodejs/index.md @@ -1,6 +1,6 @@ --- -date: 2025-01-06T10:00:00 -updatedAt: 2025-01-06T10:00:00 +date: 2025-10-12T23:40:00 +updatedAt: 2025-10-12T23:40:00 title: Reading and Writing Files in Node.js - The Complete Modern Guide slug: reading-writing-files-nodejs description: Learn the modern way to read and write files in Node.js using promises, streams, and file handles. Master memory-efficient file operations for production applications. @@ -187,7 +187,7 @@ await createBeepWav() This example demonstrates several key concepts for binary file manipulation in Node.js. The `encodeWavPcm16()` function creates a complete WAV file structure by constructing both the header and audio data sections. The WAV header contains metadata like file size, audio format, sample rate, and number of channels, while the data section contains the actual audio samples. -The `generateBeepTone()` function creates audio samples using a sine wave mathematical formula, generating a pure tone at the specified frequency. Each sample represents the amplitude of the sound wave at a specific point in time, and when played back at the correct sample rate, these digital values recreate the original analog sound. +The `makeSine()` function creates audio samples using a sine wave mathematical formula, generating a pure tone at the specified frequency. Each sample represents the amplitude of the sound wave at a specific point in time, and when played back at the correct sample rate, these digital values recreate the original analog sound. Don't worry too much about the specifics of this example dealing with the WAV binary format! What matters here is learning that we can put arbitrary binary data into a buffer and write it into a file using `writeFile()`. The key takeaway is that Node.js treats all file operations the same way, whether you're writing text, JSON, images, audio, or any other type of data. @@ -280,10 +280,6 @@ The function uses Buffer methods like `readUInt32LE()` and `readUInt16LE()` to i The duration calculation combines several pieces of metadata: we divide the total audio data size by the byte rate (bytes per second) to get the duration in seconds, then multiply by 1000 to convert to milliseconds. This demonstrates how understanding both the file format and basic math allows us to derive meaningful information from binary data. -:::note[Finding free WAV files] -You can try out this example with the beep WAV files we created in the previous example or, if you need some longer free WAV files, you can check out a website with free audio samples like [Zapsplat](https://www.zapsplat.com/). -::: - :::note[Binary file complexity] This is a very simple example showing basic binary manipulation using only Node.js built-ins. We can handle the WAV format manually here because we're creating a minimal, single-channel PCM file with known parameters. @@ -318,7 +314,7 @@ try { const [databaseConfig, apiConfig, loggingConfig, featureFlagsConfig] = await Promise.all(promises) - console.log(`Successfully loaded config files`, { + console.log('Successfully loaded config files', { databaseConfig, apiConfig, loggingConfig, @@ -341,7 +337,7 @@ This concurrent approach provides a significant performance improvement. If we u ```javascript {26-29} // write-multiple-files.js -import { writeFile } from 'node:fs/promises' +import { mkdir, writeFile } from 'node:fs/promises' async function generateReports(data) { const reports = [ @@ -385,6 +381,7 @@ const reportData = { yearly: { sales: 365000, visitors: 91250 }, } +await mkdir('reports', { recursive: true }) // Ensure reports directory exists await generateReports(reportData) ``` @@ -411,11 +408,11 @@ Learn more: [Promise.all() on MDN](https://developer.mozilla.org/en-US/docs/Web/ ### Working with Directories -Files don't exist in isolation, they live in directories. What if you need to process all files in a folder, or create directories dynamically? You'll often need to work with directories too: +In the latest example we used the `mkdir()` function to create a directory before writing files into it. This is a common pattern because files don't exist in isolation, they live in directories. What if you need to process all files in a folder, or create directories dynamically? Let's see a more complete example that combines several file system operations: ```javascript {8,11,14,15,17} // process-directory.js -import { readdir, mkdir, stat } from 'node:fs/promises' +import { mkdir, readdir, stat } from 'node:fs/promises' import { join } from 'node:path' async function processDirectory(dirPath) { @@ -513,7 +510,7 @@ try { // Synchronous file writing try { const userData = { name: 'John Doe', email: 'john@example.com' } - const jsonData = JSON.stringify(userData, null, 2) + const jsonData = JSON.stringify(userData) writeFileSync('user-data.json', jsonData, 'utf8') console.log('User data saved successfully!') } catch (error) { @@ -558,47 +555,48 @@ Imagine you're trying to read a 2GB log file using `readFile()`. Your Node.js pr 1. **Out of memory errors** - Your application might crash 2. **Poor performance** - High memory usage affects other operations -### Understanding Node.js Buffer Limits +### Understanding Memory Considerations for Large Files -Node.js has built-in limits on buffer sizes to prevent applications from consuming too much memory. You can check these limits: +While modern Node.js has significantly increased buffer limits (the theoretical maximum is now around 9 petabytes, compared to the original limits of ~1GB on 32-bit and ~2GB on 64-bit architectures), this doesn't mean we should load massive files into memory all at once. You can check the current buffer limits on your system: ```javascript // check-buffer-limits.js -// Check the maximum buffer size -console.log('Max buffer size:', Buffer.constants.MAX_LENGTH) -console.log('Max string length:', Buffer.constants.MAX_STRING_LENGTH) +import buffer from 'node:buffer' -// On most systems: -// MAX_LENGTH is around 2GB (2,147,483,647 bytes on 64-bit systems) -// MAX_STRING_LENGTH is around 1GB +// Check the maximum buffer size +console.log('Max buffer size:', buffer.constants.MAX_LENGTH) +console.log('Max string length:', buffer.constants.MAX_STRING_LENGTH) + +// Convert to more readable format +const maxSizeGB = (buffer.constants.MAX_LENGTH / (1024 * 1024 * 1024)).toFixed( + 2, +) +console.log(`That's approximately ${maxSizeGB} GB`) ``` -If you try to read a file larger than these limits using `readFile()`, you'll get an error: +Even though Node.js can theoretically handle extremely large buffers, attempting to read huge files into memory can cause practical problems: ```javascript -// handle-large-file-error.js +// handle-large-file-memory.js import { readFile } from 'node:fs/promises' try { - // This will fail if the file is larger than the buffer limit + // Even if this doesn't hit buffer limits, it might cause memory issues + console.log('Attempting to read large file...') const hugeFile = await readFile('massive-dataset.csv', 'utf8') + console.log('File loaded successfully!') } catch (error) { if (error.code === 'ERR_FS_FILE_TOO_LARGE') { - console.log('File is too large to read into memory at once!') + console.log('File exceeds buffer limits!') + } else if (error.code === 'ENOMEM') { + console.log('Not enough memory available!') + } else { + console.log('Error loading file:', error.message) } } ``` -So does this mean that Node.js can't handle big files?! Of course not, Node.js is actually quite good at handling them... we just need to use different tools to do that! The trick is to make sure we don't load ALL the data into memory in one go, but we process the data in smaller incremental chunks! - -### When to Use Promise-Based Methods - -Promise-based file operations are great when: - -- **Files are small to medium-sized** (typically under 100MB) -- **You need the entire content at once** (parsing JSON, reading config files) -- **Simplicity is important** (rapid prototyping, simple scripts) -- **Memory usage isn't a concern** (plenty of RAM available) +The key insight is that loading massive files into memory all at once is rarely a good idea, even when technically possible. It can lead to memory pressure, slower performance, and poor user experience. Instead, we should process large files incrementally using streams or file handles! ## Advanced Node.js File Operations with File Handles @@ -635,13 +633,13 @@ async function readFileInChunks(filePath) { // Process the chunk const chunk = buffer.subarray(0, result.bytesRead) - console.log(`Read ${result.bytesRead} bytes:`, chunk.toString('utf8')) + console.log(`>>> Read ${result.bytesRead} bytes:`, chunk.toString('utf8')) position += result.bytesRead totalBytesRead += result.bytesRead } - console.log(`Total bytes read: ${totalBytesRead}`) + console.log(`>>> Total bytes read: ${totalBytesRead}`) } catch (error) { console.error('Error reading file:', error.message) throw error @@ -1066,7 +1064,7 @@ We've covered a comprehensive range of file operation techniques in Node.js, fro // stream-buffer-size.js // For large files, use bigger chunks const stream = createReadStream(path, { - highWaterMark: 128 * 1024, // 64KB chunks + highWaterMark: 128 * 1024, // 128KB chunks }) ``` @@ -1133,7 +1131,7 @@ Yes! Modern Node.js supports top-level await in ES modules, so you can use `awai ### How do I process multiple files concurrently in Node.js? -Use `Promise.all()` or `Promise.allSettled()` with an array of promises to process multiple files simultaneously. For example: `await Promise.all(filenames.map(name => readFile(name)))`. This is much faster than processing files sequentially, especially for I/O-bound operations. If you are processing large files, you might want to consider using streams. You can create multiple stream objects and pipeline and run them concurrently. +Use `Promise.all()` or `Promise.allSettled()` with an array of promises to process multiple files simultaneously. For example: `await Promise.all(filenames.map(name => readFile(name)))`. This is much faster than processing files sequentially, especially for I/O-bound operations. If you are processing large files, you might want to consider using streams. You can create multiple stream pipelines and run them concurrently. --- @@ -1146,7 +1144,3 @@ If you found value in this comprehensive guide to file operations, you'll love * This book dives deep into the patterns, techniques, and best practices that separate good Node.js code from great Node.js code. From fundamental concepts like the ones covered in this article to advanced architectural patterns for building scalable applications, it provides the knowledge you need to write professional, maintainable Node.js code. **Ready to master Node.js?** Visit our [homepage](/) to discover how Node.js Design Patterns can accelerate your development journey and help you build better applications with confidence. - -``` - -``` From adf7807ea7eeb3f64af46aae7112945afb6dfb8d Mon Sep 17 00:00:00 2001 From: Luciano Mammino Date: Sun, 12 Oct 2025 23:37:55 +0200 Subject: [PATCH 7/7] chore: make prettier pretty and happy --- src/styles/global.css | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/styles/global.css b/src/styles/global.css index be2d141..a49ecc9 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -96,8 +96,8 @@ /* Semantic feedback colors */ --color-info: oklch(0.7 0.1 223); --color-info-content: oklch(0.7 0.1 223); - --color-tip: oklch(0.26 0.10 113.29); - --color-tip-content: oklch(0.90 0.04 113.29); + --color-tip: oklch(0.26 0.1 113.29); + --color-tip-content: oklch(0.9 0.04 113.29); --color-success: oklch(72% 0.17 150); --color-success-content: oklch(15% 0.05 170); --color-warning: oklch(75% 0.12 70); @@ -239,7 +239,7 @@ } .content table, -.content .highlight>pre, +.content .highlight > pre, .content pre.example { max-height: 70vh; margin: 1em 0; @@ -287,62 +287,62 @@ aside.admonition .admonition-title:before { mask-repeat: no-repeat; } -aside.admonition .admonition-content> :last-child { +aside.admonition .admonition-content > :last-child { margin-bottom: 0; } -aside.admonition .admonition-content> :first-child { +aside.admonition .admonition-content > :first-child { margin-top: 0; } /* Note admonition */ -aside.admonition[data-admonition-type="note"] { +aside.admonition[data-admonition-type='note'] { --admonition-color: var(--color-info-content); background-color: oklch(from var(--color-info) l c h / 0.1); } -aside.admonition[data-admonition-type="note"] .admonition-title::before { +aside.admonition[data-admonition-type='note'] .admonition-title::before { mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='M12 16v-4'/%3E%3Cpath d='M12 8h.01'/%3E%3C/svg%3E"); } /* Tip admonition */ -aside.admonition[data-admonition-type="tip"] { +aside.admonition[data-admonition-type='tip'] { --admonition-color: var(--color-tip-content); background-color: oklch(from var(--color-tip) l c h / 0.1); } -aside.admonition[data-admonition-type="tip"] .admonition-title::before { +aside.admonition[data-admonition-type='tip'] .admonition-title::before { mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M9 11H6l3-9 2 9'/%3E%3Cpath d='M13 11h3l-3 9-2-9'/%3E%3C/svg%3E"); } /* Important admonition */ -aside.admonition[data-admonition-type="important"] { +aside.admonition[data-admonition-type='important'] { --admonition-color: var(--color-primary); background-color: oklch(from var(--color-primary) l c h / 0.1); } -aside.admonition[data-admonition-type="important"] .admonition-title::before { +aside.admonition[data-admonition-type='important'] .admonition-title::before { mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='12' cy='12' r='10'/%3E%3Cpath d='M12 8v4'/%3E%3Cpath d='M12 16h.01'/%3E%3C/svg%3E"); } /* Caution admonition */ -aside.admonition[data-admonition-type="caution"] { +aside.admonition[data-admonition-type='caution'] { --admonition-color: var(--color-warning); background-color: oklch(from var(--color-warning) l c h / 0.1); } -aside.admonition[data-admonition-type="caution"] .admonition-title::before { +aside.admonition[data-admonition-type='caution'] .admonition-title::before { mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='M10.29 3.86L1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0z'/%3E%3Cline x1='12' y1='9' x2='12' y2='13'/%3E%3Cline x1='12' y1='17' x2='12.01' y2='17'/%3E%3C/svg%3E"); } /* Danger/Warning admonition */ -aside.admonition[data-admonition-type="danger"], -aside.admonition[data-admonition-type="warning"] { +aside.admonition[data-admonition-type='danger'], +aside.admonition[data-admonition-type='warning'] { --admonition-color: var(--color-error); background-color: oklch(from var(--color-error) l c h / 0.1); } -aside.admonition[data-admonition-type="danger"] .admonition-title::before, -aside.admonition[data-admonition-type="warning"] .admonition-title::before { +aside.admonition[data-admonition-type='danger'] .admonition-title::before, +aside.admonition[data-admonition-type='warning'] .admonition-title::before { mask-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' viewBox='0 0 24 24' fill='none' stroke='currentColor' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolygon points='7.86 2 16.14 2 22 7.86 22 16.14 16.14 22 7.86 22 2 16.14 2 7.86 7.86 2'/%3E%3Cline x1='15' y1='9' x2='9' y2='15'/%3E%3Cline x1='9' y1='9' x2='15' y2='15'/%3E%3C/svg%3E"); -} \ No newline at end of file +}