fix: concurrent file processing

Author: mdevils

src/cli/save-generation-result.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import path from 'path';
 import * as R from 'ramda';
 import {ClientGenerationResultFile} from '../schema-to-typescript/config';
+import {lock} from '../utils/lock';
 
 export async function saveGenerationResult({
     files,
@@ -27,19 +28,21 @@ export async function saveGenerationResult({
         ...files.map(async ({filename, data}) => {
             const fullFilename = path.resolve(outputDirPath, filename);
             try {
-                let exists = false;
-                try {
-                    const existingContent = await fs.promises.readFile(fullFilename, 'utf8');
-                    exists = true;
-                    if (existingContent === data) {
-                        console.log('[no change] ' + fullFilename);
-                        return;
+                await lock(`file:${fullFilename}`, async () => {
+                    let exists = false;
+                    try {
+                        const existingContent = await fs.promises.readFile(fullFilename, 'utf8');
+                        exists = true;
+                        if (existingContent === data) {
+                            console.log('[no change] ' + fullFilename);
+                            return;
+                        }
+                    } catch (e) {
+                        // ok
                     }
-                } catch (e) {
-                    // ok
-                }
-                await fs.promises.writeFile(fullFilename, data);
-                console.log(`[${exists ? 'updated' : 'created'}] ${fullFilename}`);
+                    await fs.promises.writeFile(fullFilename, data);
+                    console.log(`[${exists ? 'updated' : 'created'}] ${fullFilename}`);
+                });
             } catch (e) {
                 throw new Error(`Could not save file "${fullFilename}": ${e instanceof Error ? e.message : e}.`);
             }

src/schema-to-typescript/common/client.ts

@@ -176,7 +176,7 @@ export function generateClient({
             logicalExpression(
                 '??',
                 memberExpression(memberExpression(thisExpression(), identifier('constructor')), identifier('name')),
-                stringLiteral('name')
+                stringLiteral(name)
             )
         ),
         objectProperty(identifier('baseUrl'), stringLiteral(baseUrl ?? servers[0]?.url ?? defaultServerUrl)),

src/utils/lock.ts

@@ -0,0 +1,16 @@
+const lockedKeys: Record<string, Promise<unknown>> = {};
+
+export function lock<T>(keyName: string, callback: () => Promise<T>): Promise<T> {
+    const promise = (lockedKeys[keyName] ?? Promise.resolve())
+        .then(
+            () => callback(),
+            () => callback()
+        )
+        .finally(() => {
+            if (lockedKeys[keyName] === promise) {
+                delete lockedKeys[keyName];
+            }
+        });
+    lockedKeys[keyName] = promise;
+    return promise;
+}

src/utils/postprocess-files.ts

@@ -1,12 +1,27 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import type {ESLint as ESLintClass} from 'eslint';
+import {lock} from './lock';
 import {makeDir} from './make-dir';
 import {
     ClientGenerationResultFile,
     CommonOpenApiClientGeneratorConfigPostprocess
 } from '../schema-to-typescript/config';
 
+let eslintInstance: ESLintClass | null = null;
+
+function getEslintInstance(): ESLintClass {
+    if (eslintInstance === null) {
+        // This is an optional dependency, so we require it here to avoid loading it when it's not needed.
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const {ESLint} = require('eslint') as {ESLint: typeof ESLintClass};
+        eslintInstance = new ESLint({
+            fix: true
+        });
+    }
+    return eslintInstance;
+}
+
 export async function postprocessFiles({
     files,
     config: {eslint: enableEslint} = {},
@@ -25,68 +40,72 @@ export async function postprocessFiles({
             directories.add(directory);
         }
 
-        for (const directory of Array.from(directories)) {
-            try {
-                await fs.stat(directory);
-            } catch (_e) {
-                const directoryBits = directory.split(path.sep);
-                let currentDirectory = directoryBits.shift() || '/';
-                for (;;) {
-                    try {
-                        await fs.stat(currentDirectory);
-                    } catch (e) {
-                        await makeDir(currentDirectory);
-                        directoriesToRemove.unshift(currentDirectory);
-                    }
-                    const subDirectory = directoryBits.shift();
-                    if (!subDirectory) {
-                        break;
+        await lock('calc:directories', async () => {
+            for (const directory of Array.from(directories)) {
+                try {
+                    await fs.stat(directory);
+                } catch (_e) {
+                    const directoryBits = directory.split(path.sep);
+                    let currentDirectory = directoryBits.shift() || '/';
+                    for (;;) {
+                        try {
+                            await fs.stat(currentDirectory);
+                        } catch (e) {
+                            await makeDir(currentDirectory);
+                            directoriesToRemove.unshift(currentDirectory);
+                        }
+                        const subDirectory = directoryBits.shift();
+                        if (!subDirectory) {
+                            break;
+                        }
+                        currentDirectory = path.join(currentDirectory, subDirectory);
                     }
-                    currentDirectory = path.join(currentDirectory, subDirectory);
                 }
             }
-        }
+        });
 
+        const eslint = getEslintInstance();
         try {
-            // This is an optional dependency, so we require it here to avoid loading it when it's not needed.
-            // eslint-disable-next-line @typescript-eslint/no-var-requires
-            const {ESLint} = require('eslint') as {ESLint: typeof ESLintClass};
-            const eslint = new ESLint({
-                fix: true
-            });
-
             return await Promise.all(
                 files.map(async (file) => {
                     const filePath = path.resolve(outputDirPath, file.filename);
-                    let fileCreated = false;
-                    try {
+                    return await lock(`file:${filePath}`, async () => {
+                        let fileCreated = false;
                         try {
-                            await fs.stat(filePath);
-                        } catch (_e) {
-                            await fs.writeFile(filePath, file.data);
-                            fileCreated = true;
-                        }
-                        const [result] = await eslint.lintText(file.data, {filePath});
-                        for (const message of result.messages) {
-                            if (message.fatal) {
-                                throw new Error(`Fatal ESLint error in ${file.filename}: ${message.message}`);
+                            try {
+                                await fs.stat(filePath);
+                            } catch (_e) {
+                                await fs.writeFile(filePath, file.data);
+                                fileCreated = true;
+                            }
+                            const [result] = await eslint.lintText(file.data, {filePath});
+                            for (const message of result.messages) {
+                                if (message.fatal) {
+                                    throw new Error(`Fatal ESLint error in ${file.filename}: ${message.message}`);
+                                }
+                            }
+                            return {
+                                ...file,
+                                data: result.output ?? file.data
+                            };
+                        } finally {
+                            if (fileCreated) {
+                                await fs.unlink(filePath);
                             }
                         }
-                        return {
-                            ...file,
-                            data: result.output ?? file.data
-                        };
-                    } finally {
-                        if (fileCreated) {
-                            await fs.unlink(filePath);
-                        }
-                    }
+                    });
                 })
             );
         } finally {
-            for (const directory of directoriesToRemove) {
-                await fs.rmdir(directory);
-            }
+            await lock('cleanup:directories', async () => {
+                for (const directory of directoriesToRemove) {
+                    try {
+                        await fs.rmdir(directory);
+                    } catch (e) {
+                        // Ignore
+                    }
+                }
+            });
         }
     }
     return files;