Better debugging of failed scans during ci

Author: J12934

tests/integration/helpers.js

@@ -5,6 +5,7 @@ kc.loadFromDefault();
 
 const k8sCRDApi = kc.makeApiClient(k8s.CustomObjectsApi);
 const k8sBatchApi = kc.makeApiClient(k8s.BatchV1Api);
+const k8sPodsApi = kc.makeApiClient(k8s.CoreV1Api);
 
 const namespace = "integration-tests";
 
@@ -33,28 +34,65 @@ async function getScan(name) {
   return scan;
 }
 
+async function displayAllLogsForJob(jobName) {
+  console.log(`Listing logs for Job ${jobName}:`);
+  const {
+    body: { items: pods },
+  } = await k8sPodsApi.listNamespacedPod(
+    "default",
+    true,
+    undefined,
+    undefined,
+    undefined,
+    `job-name=${jobName}`
+  );
+
+  for (const pod of pods) {
+    for (const container of pod.spec.containers) {
+      const response = await k8sPodsApi.readNamespacedPodLog(
+        pod.metadata.name,
+        "default",
+        container.name
+      );
+      console.log(`Container ${container.name}:`);
+      console.log(response.body);
+    }
+  }
+}
+
 async function logJobs() {
   try {
     const { body: jobs } = await k8sBatchApi.listNamespacedJob(namespace);
 
+    console.log("Logging spec & status of jobs in namespace");
+
     for (const job of jobs.items) {
       console.log(`Job: '${job.metadata.name}' Spec:`);
       console.dir(job.spec);
       console.log(`Job: '${job.metadata.name}' Status:`);
       console.dir(job.status);
+
+      await displayAllLogsForJob(job.metadata.name);
     }
   } catch (error) {
     console.info(`Failed to list Jobs'`);
   }
 }
 
+async function disasterRecovery(scanName) {
+  const scan = await getScan(scanName);
+  console.error("Last Scan State:");
+  console.dir(scan);
+  await logJobs();
+}
+
 /**
  *
  * @param {string} name name of the scan. Actual name will be sufixed with a random number to avoid conflicts
  * @param {string} scanType type of the scan. Must match the name of a ScanType CRD
  * @param {string[]} parameters cli argument to be passed to the scanner
  * @param {number} timeout in seconds
- * @returns {scan.findings} returns findings { categories, severities, count } 
+ * @returns {scan.findings} returns findings { categories, severities, count }
  */
 async function scan(name, scanType, parameters = [], timeout = 180) {
   const scanDefinition = {
@@ -88,19 +126,16 @@ async function scan(name, scanType, parameters = [], timeout = 180) {
       await deleteScan(actualName);
       return status.findings;
     } else if (status && status.state === "Errored") {
-      await deleteScan(actualName);
+      console.error("Scan Errored");
+      await disasterRecovery(actualName);
+
       throw new Error(
         `Scan failed with description "${status.errorDescription}"`
       );
     }
   }
-
   console.error("Scan Timed out!");
-
-  const scan = await getScan(actualName);
-  console.log("Last Scan State:");
-  console.dir(scan);
-  await logJobs();
+  await disasterRecovery(actualName);
 
   throw new Error("timed out while waiting for scan results");
 }