test: fix flakey snapshot normalization logic (#7203)

serhalp · web-flow · commit 1b00c7789780 · 2025-04-14T17:22:55.000Z
* test: fix flakey snapshot normalization logic This logic was normalizing integration test snapshots by replacing any sequence of five digits with `88888`. The intent is to handle some randomness in test output. It wasn't working some percentage of the time because randomly selected ports are sometimes _four_ digits. You can see an example in this failed run: https://github.com/netlify/cli/actions/runs/14434953424/job/40474516002?pr=7202#step:10:1143. While I was at it, I made it less much less generic and moved it into the only test suite that relies on it. * test: make normalized snapshot port more explicit
diff --git a/tests/integration/__snapshots__/framework-detection.test.js.snap b/tests/integration/__snapshots__/framework-detection.test.js.snap
@@ -9,11 +9,11 @@ exports[`frameworks/framework-detection > should default to process.cwd() and st
 ◈ Running static server from \\"should-default-to-process-cwd-and-static-server\\"
 ◈ Setting up local development server
 
-◈ Static server listening to 88888
+◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
 
    ┌──────────────────────────────────────────────────┐
    │                                                  │
-   │   ◈ Server now ready on http://localhost:88888   │
+   │   ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED>   │
    │                                                  │
    └──────────────────────────────────────────────────┘"
 `;
@@ -38,11 +38,11 @@ exports[`frameworks/framework-detection > should filter frameworks with no dev c
 ◈ Running static server from \\"should-filter-frameworks-with-no-dev-command\\"
 ◈ Setting up local development server
 
-◈ Static server listening to 88888
+◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
 
    ┌──────────────────────────────────────────────────┐
    │                                                  │
-   │   ◈ Server now ready on http://localhost:88888   │
+   │   ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED>   │
    │                                                  │
    └──────────────────────────────────────────────────┘"
 `;
@@ -150,11 +150,11 @@ exports[`frameworks/framework-detection > should use static server when --dir fl
 ◈ Running static server from \\"should-use-static-server-when-dir-flag-is-passed/public\\"
 ◈ Setting up local development server
 
-◈ Static server listening to 88888
+◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
 
    ┌──────────────────────────────────────────────────┐
    │                                                  │
-   │   ◈ Server now ready on http://localhost:88888   │
+   │   ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED>   │
    │                                                  │
    └──────────────────────────────────────────────────┘"
 `;
@@ -168,11 +168,11 @@ exports[`frameworks/framework-detection > should use static server when framewor
 ◈ Running static server from \\"should-use-static-server-when-framework-is-set-to-static\\"
 ◈ Setting up local development server
 
-◈ Static server listening to 88888
+◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
 
    ┌──────────────────────────────────────────────────┐
    │                                                  │
-   │   ◈ Server now ready on http://localhost:88888   │
+   │   ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED>   │
    │                                                  │
    └──────────────────────────────────────────────────┘"
 `;
@@ -185,11 +185,11 @@ exports[`frameworks/framework-detection > should warn if using static server and
 ◈ Running static server from \\"should-warn-if-using-static-server-and-target-port-is-configured/public\\"
 ◈ Setting up local development server
 
-◈ Static server listening to 88888
+◈ Static server listening to <SNAPSHOT_PORT_NORMALIZED>
 
    ┌──────────────────────────────────────────────────┐
    │                                                  │
-   │   ◈ Server now ready on http://localhost:88888   │
+   │   ◈ Server now ready on http://localhost:<SNAPSHOT_PORT_NORMALIZED>   │
    │                                                  │
    └──────────────────────────────────────────────────┘"
 `;
diff --git a/tests/integration/framework-detection.test.js b/tests/integration/framework-detection.test.js
@@ -10,6 +10,12 @@ import { normalize } from './utils/snapshots.js'
 
 const content = 'Hello World!'
 
+// Normalize random ports
+const normalizeSnapshot = (output, opts) =>
+  normalize(output, opts)
+    .replaceAll(/localhost:\d+/g, 'localhost:<SNAPSHOT_PORT_NORMALIZED>')
+    .replaceAll(/listening to \d+/g, 'listening to <SNAPSHOT_PORT_NORMALIZED>')
+
 describe.concurrent('frameworks/framework-detection', () => {
   test('should default to process.cwd() and static server', async (t) => {
     await withSiteBuilder(t, async (builder) => {
@@ -25,7 +31,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         const responseContent = await response.text()
 
         t.expect(responseContent).toEqual(content)
-        t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
+        t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
       })
     })
   })
@@ -44,7 +50,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         const responseContent = await response.text()
 
         t.expect(responseContent).toEqual(content)
-        t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
+        t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
       })
     })
   })
@@ -64,7 +70,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         const responseContent = await response.text()
 
         t.expect(responseContent).toEqual(content)
-        t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
+        t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
       })
     })
   })
@@ -85,7 +91,7 @@ describe.concurrent('frameworks/framework-detection', () => {
           const responseContent = await response.text()
 
           t.expect(responseContent).toEqual(content)
-          t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
+          t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
         },
       )
     })
@@ -102,7 +108,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         true,
       ).catch((error_) => error_)
 
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -112,7 +118,7 @@ describe.concurrent('frameworks/framework-detection', () => {
 
       // a failure is expected since this is not a true create-react-app project
       const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -121,7 +127,7 @@ describe.concurrent('frameworks/framework-detection', () => {
       await builder.withNetlifyToml({ config: { dev: { framework: 'to-infinity-and-beyond-js' } } }).build()
 
       const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -135,7 +141,7 @@ describe.concurrent('frameworks/framework-detection', () => {
 
       // a failure is expected since this is not a true create-react-app project
       const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -148,7 +154,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         () => {},
         true,
       ).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -161,7 +167,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         () => {},
         true,
       ).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -174,7 +180,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         () => {},
         true,
       ).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -198,7 +204,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         true,
       ).catch((error_) => error_)
 
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -232,7 +238,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         await childProcess
       }
       const error = await asyncErrorBlock().catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -259,7 +265,7 @@ describe.concurrent('frameworks/framework-detection', () => {
       }
       const error = await asyncErrorBlock().catch((error_) => error_)
       t.expect(
-        normalize(error.stdout, { duration: true, filePath: true }).includes(
+        normalizeSnapshot(error.stdout, { duration: true, filePath: true }).includes(
           'Detected commands for: Gatsby, Create React App. Update your settings to specify which to use. Refer to https://ntl.fyi/dev-monorepo for more information.',
         ),
       )
@@ -278,7 +284,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         true,
       ).catch((error_) => error_)
 
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -299,7 +305,7 @@ describe.concurrent('frameworks/framework-detection', () => {
         const responseContent = await response.text()
 
         t.expect(responseContent).toEqual(content)
-        t.expect(normalize(output, { duration: true, filePath: true })).toMatchSnapshot()
+        t.expect(normalizeSnapshot(output, { duration: true, filePath: true })).toMatchSnapshot()
       })
     })
   })
@@ -317,7 +323,7 @@ describe.concurrent('frameworks/framework-detection', () => {
 
       // a failure is expected since this is not a true Gatsby project
       const error = await withDevServer({ cwd: builder.directory }, () => {}, true).catch((error_) => error_)
-      t.expect(normalize(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
+      t.expect(normalizeSnapshot(error.stdout, { duration: true, filePath: true })).toMatchSnapshot()
     })
   })
 
@@ -395,7 +401,7 @@ describe.concurrent('frameworks/framework-detection', () => {
           const responseJson = await response.json()
           t.expect(responseJson).toStrictEqual({ CONTEXT_CHECK: 'PRODUCTION' })
 
-          const normalizedText = normalize(output, { duration: true, filePath: true })
+          const normalizedText = normalizeSnapshot(output, { duration: true, filePath: true })
           t.expect(
             normalizedText.includes(
               `Changes will not be hot-reloaded, so if you need to rebuild your site you must exit and run 'netlify serve' again`,
diff --git a/tests/integration/utils/snapshots.js b/tests/integration/utils/snapshots.js
@@ -2,8 +2,6 @@ const baseNormalizers = [
   // Information about the package and the OS
   { pattern: /netlify-cli\/.+node-.+/g, value: 'netlify-cli/test-version test-os test-node-version' },
   { pattern: /@netlify\/build (\d+\.\d+\.\d+)/g, value: '@netlify/build 0.0.0' },
-  // normalize random ports
-  { pattern: /\d{5}/g, value: '88888' },
   // windows specific
   { pattern: /\\/gu, value: '/' },
   { pattern: /\r\n/gu, value: '\n' },
