Fix script to get correct broken links

broken-links-script/cypress.config.cjs

@@ -6,10 +6,17 @@ module.exports = defineConfig({
     setupNodeEvents(on, config) {
       // implement node event listeners here
     },
+    retries: {
+      runMode: 3,
+      openMode: 0   
+    },
     chromeWebSecurity: false, 
-    pageLoadTimeout: 300000,
+    pageLoadTimeout: 20000,
     defaultCommandTimeout: 20000,
-    requestTimeout: 15000,
+    requestTimeout: 20000,
+    responseTimeout: 20000,
+    execTimeout: 20000,
+    taskTimeout: 20000,
     numTestsKeptInMemory: 0,
     experimentalMemoryManagement: true,
     viewportWidth: 1280,

broken-links-script/cypress/e2e/link-checker.cy.js

@@ -65,12 +65,20 @@ describe('Link and Routing Validation - Individual URL Checks', () => {
   urls.forEach((item) => {
     it(`should validate URL: ${item.link}`, () => {
       const url = item.link;
+      const excludedLinks = [
+      "https://de.mathworks.com/help/predmaint/ug/remaining-useful-life-estimation-using-convolutional-neural-network.html",
+      "https://medium.com/@polanitzer/prediction-of-remaining-useful-life-of-an-engine-based-on-sensors-building-a-random-forest-in-ffad82c8a1c6"
+    ];
+    if (excludedLinks.includes(url)) {
+      cy.log(`Skipping excluded link: ${url}`);
+      return;
+    }
       const fragment = url.includes('#') ? url.split('#').slice(-1)[0] : null;
-      const isCodexPage = url.includes('codex/#/');
+      const isCodexPage = url.includes('/codex/');
       const isApiPage = url.includes('/api/');
       const isGithubPage = url.includes('github.com');
+      const isGithubBlobLine = url.includes('github.com') && /\/blob\/[^#]+#L\d+(-L\d+)?$/.test(url);
       const nonHtmlExtensions = ['.txt','.json','.pdf','.zip','.csv','.xml','.not','.bin','.dat','.tar','.gz','.rar','.xsd','.yaml','.pot'];
-
       const hasNonHtmlExtension = nonHtmlExtensions.some(ext => url.endsWith(ext));
       const isNonHtmlResource = hasNonHtmlExtension || url.includes('/files/') || url.includes('/downloads/');
       const isNpmPackagePage = url.startsWith('https://www.npmjs.com/package/');
@@ -112,19 +120,20 @@ describe('Link and Routing Validation - Individual URL Checks', () => {
       }
 
       if (isCodexPage) {
-        cy.visit(url, { timeout: 50000 });
+        cy.visit(url, { timeout: 20000 });
 
-        cy.get('[data-cy="c8y-title--title-outlet"] .text-truncate', { timeout: 50000 })
+        cy.get('[data-cy="c8y-title--title-outlet"] .text-truncate', { timeout: 30000 })
           .invoke('text')
           .should('not.be.empty')
+          .and('not.match', /404 not found/i);
 
         cy.url().should('eq', url);
 
         if (fragment) {
           if (fragment.startsWith('/')) {
             cy.location('hash').should('eq', `#${fragment}`, `URL hash should match the fragment: #${fragment}`);
           } else {
-            cy.get(`#${fragment}`, { timeout: 10000 })
+            cy.get(`#${fragment}`, { timeout: 20000 })
               .should('exist', `Fragment "${fragment}" does not exist on the page`)
               .then(() => {
                 cy.document().then((doc) => {
@@ -136,17 +145,35 @@ describe('Link and Routing Validation - Individual URL Checks', () => {
         }
       }
       else if (isApiPage) {
-        cy.visit(url, { timeout: 50000 });
+        cy.visit(url, { timeout: 20000 });
         if (fragment) {
-          cy.get(`[id="${fragment}"]`, { timeout: 10000 }).should('exist');
+          cy.get(`[id="${fragment}"]`, { timeout: 20000 }).should('exist');
+        }
+      }
+      else if (isGithubBlobLine) {
+        const baseUrl = url.split('#')[0];
+        const match = url.match(/#L(\d+)/);
+        const lineNumber = match ? match[1] : null;
+        cy.request({ url: baseUrl, failOnStatusCode: false }).then((res) => {
+          expect(res.status, `GitHub blob file should exist: ${baseUrl}`)
+            .to.be.oneOf([200, 301, 302]);
+        });
+        if (lineNumber) {
+          cy.visit(baseUrl, { timeout: 30000 });
+          const selector = `#L${lineNumber}, #LC${lineNumber}`;
+          cy.get('body').then(($body) => {
+            if ($body.find(selector).length === 0) {
+              throw new Error(`Line ${lineNumber} does NOT exist in ${baseUrl}`);
+            }
+          });
         }
       }
       else if (isGithubPage && fragment) {
-        cy.visit(url, { timeout: 50000 });
+        cy.visit(url, { timeout: 20000 });
         checkGithubFragment(fragment);
       }
       else if (fragment) {
-        cy.visit(url, { timeout: 50000, failOnStatusCode: false, headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/122.0 Safari/537.36' }});
+        cy.visit(url, { timeout: 20000, failOnStatusCode: false, headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/122.0 Safari/537.36' }});
         checkRegularFragment(fragment);
       }
       else {
@@ -160,7 +187,7 @@ describe('Link and Routing Validation - Individual URL Checks', () => {
             expect(response.status).to.be.oneOf([200, 201, 202, 203, 204, 301, 302, 304]);
             expect(response.body).not.to.be.empty;
           } else {
-            cy.visit(url, { timeout: 50000 });
+            cy.visit(url, { timeout: 20000 });
             cy.document().its('body').should('not.be.empty');
           }
         });

broken-links-script/cypress/support/e2e.js

@@ -75,7 +75,7 @@ Cypress.on('uncaught:exception', (err) => {
     if (err.message.includes("Script error for 'hubspot'")) {
       return false;
     }
-    if (err.message.includes("Stop the wrapper script on User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/140.0.0.0 Safari/537.36")) {
+    if (err.message.includes("Stop the wrapper script on User-Agent")) {
       return false;
     }
     if (err.message.includes("Failed to construct 'Response': Response with null body status cannot have body")) {
@@ -90,6 +90,9 @@ Cypress.on('uncaught:exception', (err) => {
     if (err.message.includes("$ is not defined")) {
       return false;
     }
+    if (err.message.includes("bootstrap is not defined")) {
+      return false;
+    }
 
   });