Test result report missing logs

sunmou99 · web-flow · commit 730f73e56c4d · 2021-07-14T22:13:22.000Z
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -314,6 +314,14 @@ jobs:
             --noadd_timestamp \
             --short_output_paths \
             ${additional_flags[*]}
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then
+            # No summary was created, make a placeholder one.
+            echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
+          fi
       - name: Upload Desktop integration tests artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
@@ -442,6 +450,13 @@ jobs:
             --noadd_timestamp \
             --short_output_paths \
             ${additional_flags[*]} 
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f build-results-android-${{ matrix.os }}.log.json ]; then
+            echo "__SUMMARY_MISSING__" > build-results-android-${{ matrix.os }}.log.json
+          fi
       - name: Upload Android integration tests artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
@@ -540,6 +555,13 @@ jobs:
             --noadd_timestamp \
             --short_output_paths \
             ${additional_flags[*]} 
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f build-results-ios-macos-latest.log.json ]; then
+            echo "__SUMMARY_MISSING__" > build-results-ios-macos-latest.log.json
+          fi
       - name: Upload iOS integration tests artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
@@ -637,6 +659,13 @@ jobs:
             --noadd_timestamp \
             --short_output_paths \
             ${additional_flags[*]} 
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f build-results-tvos-macos-latest.log.json ]; then
+            echo "__SUMMARY_MISSING__" > build-results-tvos-macos-latest.log.json
+          fi
       - name: Upload tvOS integration tests artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
@@ -704,6 +733,13 @@ jobs:
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" --artifact testapps
       - name: Run Desktop integration tests
         run: python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}"
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then
+            echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
+          fi
       - name: Upload Desktop test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
@@ -782,6 +818,13 @@ jobs:
             --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}" \
             --code_platform cpp \
             --key_file scripts/gha-encrypted/gcs_key_file.json
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json" ]; then
+            echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json"
+          fi
       - name: Upload Android test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
@@ -859,6 +902,13 @@ jobs:
             --logfile_name "ios-macos-latest-${{ matrix.ios_device }}" \
             --code_platform cpp \
             --key_file scripts/gha-encrypted/gcs_key_file.json
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json" ]; then
+            echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json"
+          fi
       - name: Upload iOS test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
@@ -920,6 +970,13 @@ jobs:
             --tvos_device "${{ matrix.tvos_device }}" \
             --logfile_name "tvos-macos-latest-${{ matrix.tvos_device }}" \
             --ci
+      - name: Prepare results summary artifact
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          if [ ! -f "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json" ]; then
+            echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json"
+          fi
       - name: Upload tvOS test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
diff --git a/scripts/gha/summarize_test_results.py b/scripts/gha/summarize_test_results.py
@@ -22,7 +22,8 @@
 
 | Failures   |                    Configs                         |
 |------------|----------------------------------------------------|
-| messaging  |[BUILD] [ERROR] [Windows] [boringssl]               |
+| missing_log|[BUILD] [ERROR] [Windows] [boringssl]               |
+| messaging  |[BUILD] [ERROR] [Windows] [openssl]                 |
 |            |[TEST] [ERROR] [Android] [All os] [emulator_min]    |
 |            |[TEST] [FAILURE] [Android] [macos] [emulator_target]|
 |            |▼(1 failed tests)                                   |
@@ -86,6 +87,7 @@
 
 TESTAPPS_HEADER = "Failures"
 CONFIGS_HEADER = "Configs"
+MISSING_LOG = "missing_log"
 
 LOG_HEADER = "INTEGRATION TEST FAILURES"
 
@@ -110,6 +112,13 @@ def print_table(log_results,
   output_lines.append(("|" + "-%s-|" * len(headers)) %
                       tuple([ re.sub("[^|]","-", header) for header in headers ]))
 
+  if MISSING_LOG in log_results.keys():
+    columns = [
+      re.sub(r'\b \b', space_char, MISSING_LOG.ljust(testapps_width)),
+      format_result(log_results.pop(MISSING_LOG), space_char=space_char, list_separator=list_separator, justify=configs_width)
+    ]
+    output_lines.append(("|" + " %s |" * len(headers)) % tuple(columns))
+  
   # Iterate through platforms and print out table lines.
   for testapp in sorted(log_results.keys()):
     columns = [
@@ -195,27 +204,37 @@ def summarize_logs(dir, markdown=False, github_log=False):
   log_data = {}
   # log_data format:
   #   { testapps: {"build": [configs]},
-  #                "test": {"errors": [configs]},
+  #               {"test": {"errors": [configs]},
   #                        {"failures": {failed_test: [configs]}}}}
   for build_log_file in build_log_files:
     configs = get_configs_from_file_name(build_log_file, build_log_name_re)
     with open(build_log_file, "r") as log_reader:
-      log_reader_data = json.loads(log_reader.read())
-      for (testapp, error) in log_reader_data["errors"].items():
+      log_text = log_reader.read()
+      if "__SUMMARY_MISSING__" in log_text:
         any_failures = True
-        log_data.setdefault(testapp, {}).setdefault("build", []).append(configs)
+        log_data.setdefault(MISSING_LOG, {}).setdefault("build", []).append(configs)
+      else:
+        log_reader_data = json.loads(log_text)
+        for (testapp, error) in log_reader_data["errors"].items():
+          any_failures = True
+          log_data.setdefault(testapp, {}).setdefault("build", []).append(configs)
 
   for test_log_file in test_log_files:
     configs = get_configs_from_file_name(test_log_file, test_log_name_re)
     with open(test_log_file, "r") as log_reader:
-      log_reader_data = json.loads(log_reader.read())
-      for (testapp, error) in log_reader_data["errors"].items():
+      log_text = log_reader.read()
+      if "__SUMMARY_MISSING__" in log_text:
         any_failures = True
-        log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
-      for (testapp, failures) in log_reader_data["failures"].items():
-        for (test, failure) in failures["failed_tests"].items():
+        log_data.setdefault(MISSING_LOG, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
+      else:
+        log_reader_data = json.loads(log_text)
+        for (testapp, error) in log_reader_data["errors"].items():
           any_failures = True
-          log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("failures", {}).setdefault(test, []).append(configs)
+          log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
+        for (testapp, failures) in log_reader_data["failures"].items():
+          for (test, failure) in failures["failed_tests"].items():
+            any_failures = True
+            log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("failures", {}).setdefault(test, []).append(configs)
 
   # log_results format:
   #   { testapps: {configs: [failed tests]} }
