Update build & test summary report (#419)

Update build & test summary report

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -221,8 +221,8 @@ jobs:
           fi
           mobile_test_on=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k mobile_test_on -o "${{github.event.inputs.mobile_test_on}}")
           echo "::set-output name=mobile_test_on::${mobile_test_on}"
-          echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k android_device -d ${mobile_test_on} )"
-          echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k ios_device -d ${mobile_test_on} )"
+          echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k android_device -t ${mobile_test_on} )"
+          echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k ios_device -t ${mobile_test_on} )"
           if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
             echo "::warning ::Running on https://github.com/${{github.repository}}/commits/$GITHUB_SHA"
             echo "::set-output name=github_ref::$GITHUB_SHA"
@@ -354,7 +354,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: log-artifact
-          path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log
+          path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}*
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Set CLOUDSDK_PYTHON (Windows)
         shell: bash
@@ -510,7 +510,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: log-artifact
-          path: build-results-android-${{ matrix.os }}.log
+          path: build-results-android-${{ matrix.os }}*
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
@@ -635,7 +635,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: log-artifact
-          path: build-results-ios-macos-latest.log
+          path: build-results-ios-macos-latest*
           retention-days: ${{ env.artifactRetentionDays }}
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
@@ -692,7 +692,7 @@ jobs:
     name: test-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}
     needs: [check_trigger, prepare_matrix, build_desktop]
     runs-on: ${{ matrix.os }}
-    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'Desktop') && needs.prepare_matrix.outputs.apis != '' && !cancelled() && !failure()
+    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'Desktop') && needs.prepare_matrix.outputs.apis != '' && !cancelled()
     strategy:
       fail-fast: false
       matrix:
@@ -722,7 +722,7 @@ jobs:
         uses: actions/upload-artifact@v2.2.2
         with:
           name: log-artifact
-          path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log
+          path: testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}*
           retention-days: ${{ env.artifactRetentionDays }}          
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
@@ -779,7 +779,7 @@ jobs:
     name: test-android-${{ matrix.build_os }}-${{ matrix.android_device }}
     needs: [check_trigger, prepare_matrix, build_android]
     runs-on: macos-latest
-    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'Android') && needs.prepare_matrix.outputs.apis != '' && !cancelled() && !failure()
+    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'Android') && needs.prepare_matrix.outputs.apis != '' && !cancelled()
     strategy:
       fail-fast: false
       matrix:
@@ -800,35 +800,34 @@ jobs:
           python-version: ${{ env.pythonVersion }}
       - name: Install python deps
         run: pip install -r scripts/gha/requirements.txt
-      - id: get-device
+      - id: get-device-type
         run: |
-          device=$(echo "${{ matrix.android_device }}" | cut -d ":" -f2)
-          echo "::set-output name=device::$device"
+          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.android_device }} )"
       - name: Run Android integration tests on Emulator locally
-        if: startsWith(matrix.android_device, 'virtual')
+        if: steps.get-device-type.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
-            --android_device "${{steps.get-device.outputs.device}}" \
-            --logfile_name "android-${{ matrix.build_os }}-${{steps.get-device.outputs.device}}" \
+            --android_device "${{ matrix.android_device }}" \
+            --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}" \
             --ci
       - name: Install Cloud SDK
-        if: startsWith(matrix.android_device, 'real')
+        if: steps.get-device-type.outputs.device_type == 'real'
         uses: google-github-actions/setup-gcloud@master
       - name: Run Android integration tests on Real Device via FTL
-        if: startsWith(matrix.android_device, 'real')
+        if: steps.get-device-type.outputs.device_type == 'real'
         run: |
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
           python scripts/gha/test_lab.py --testapp_dir testapps \
-            --android_device "${{steps.get-device.outputs.device}}" \
-            --logfile_name "android-${{ matrix.build_os }}-${{steps.get-device.outputs.device}}" \
+            --android_device "${{ matrix.android_device }}" \
+            --logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}" \
             --code_platform cpp \
             --key_file scripts/gha-encrypted/gcs_key_file.json
       - name: Upload Android test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
         with:
           name: log-artifact
-          path: testapps/test-results-android-${{ matrix.build_os }}-${{steps.get-device.outputs.device}}.log
+          path: testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}*
           retention-days: ${{ env.artifactRetentionDays }}  
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
@@ -876,7 +875,7 @@ jobs:
         if: ${{ !cancelled() }}
         shell: bash
         run: |
-          cat "testapps/test-results-android-${{ matrix.build_os }}-${{steps.get-device.outputs.device}}.log"
+          cat "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log"
           if [[ "${{ job.status }}" != "success" ]]; then
             exit 1
           fi
@@ -885,7 +884,7 @@ jobs:
     name: test-ios-macos-latest-${{ matrix.ios_device }}
     needs: [check_trigger, prepare_matrix, build_ios]
     runs-on: macos-latest
-    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'iOS') && needs.prepare_matrix.outputs.apis != '' && !cancelled() && !failure()
+    if: contains(needs.prepare_matrix.outputs.matrix_platform, 'iOS') && needs.prepare_matrix.outputs.apis != '' && !cancelled()
     strategy:
       fail-fast: false
       matrix:
@@ -905,35 +904,34 @@ jobs:
           python-version: ${{ env.pythonVersion }}
       - name: Install python deps
         run: pip install -r scripts/gha/requirements.txt
-      - id: get-device
+      - id: get-device-type
         run: |
-          device=$(echo "${{ matrix.ios_device }}" | cut -d ":" -f2)
-          echo "::set-output name=device::$device"
+          echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -d -k ${{ matrix.ios_device }} )"
       - name: Run iOS integration tests on Simulator locally
-        if: startsWith(matrix.ios_device, 'virtual')
+        if: steps.get-device-type.outputs.device_type == 'virtual'
         run: |
           python scripts/gha/test_simulator.py --testapp_dir testapps \
-            --ios_device "${{steps.get-device.outputs.device}}" \
-            --logfile_name "ios-macos-latest-${{steps.get-device.outputs.device}}" \
+            --ios_device "${{ matrix.ios_device }}" \
+            --logfile_name "ios-macos-latest-${{ matrix.ios_device }}" \
             --ci
       - name: Install Cloud SDK
-        if: startsWith(matrix.ios_device, 'real')
+        if: steps.get-device-type.outputs.device_type == 'real'
         uses: google-github-actions/setup-gcloud@master
       - name: Run iOS integration tests on Real Device via FTL
-        if: startsWith(matrix.ios_device, 'real')
+        if: steps.get-device-type.outputs.device_type == 'real'
         run: |
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
           python scripts/gha/test_lab.py --testapp_dir testapps \
-            --ios_device "${{steps.get-device.outputs.device}}" \
-            --logfile_name "ios-macos-latest-${{steps.get-device.outputs.device}}" \
+            --ios_device "${{ matrix.ios_device }}" \
+            --logfile_name "ios-macos-latest-${{ matrix.ios_device }}" \
             --code_platform cpp \
             --key_file scripts/gha-encrypted/gcs_key_file.json
       - name: Upload Android test results artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v2.2.2
         with:
           name: log-artifact
-          path: testapps/test-results-ios-macos-latest-${{steps.get-device.outputs.device}}.log
+          path: testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}*
           retention-days: ${{ env.artifactRetentionDays }}  
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
@@ -981,7 +979,7 @@ jobs:
         if: ${{ !cancelled() }}
         shell: bash
         run: |
-          cat "testapps/test-results-ios-macos-latest-${{steps.get-device.outputs.device}}.log"
+          cat "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log"
           if [[ "${{ job.status }}" != "success" ]]; then
             exit 1
           fi
@@ -1061,7 +1059,7 @@ jobs:
             ### ❌  Integration test FAILED
             Requested by @${{github.actor}} on commit ${{needs.prepare_matrix.outputs.github_ref}}
             Last updated: ${{ steps.get-time.outputs.time }}
-            **[View integration test results](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
+            **[Download integration test detailed logs and artifacts](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})**
             ${{ env.SUMMARY_TABLE }}
           GITHUB_TOKEN: ${{ github.token }}
           COMMENT_IDENTIFIER: ${{ env.statusCommentIdentifier }}

scripts/gha/build_testapps.py

@@ -80,6 +80,7 @@
 import shutil
 import subprocess
 import sys
+import json
 
 from absl import app
 from absl import flags
@@ -255,7 +256,7 @@ def main(argv):
           short_output_paths=FLAGS.short_output_paths)
       logging.info("END building for %s", testapp)
 
-  _collect_integration_tests(testapps, root_output_dir, FLAGS.artifact_name)
+  _collect_integration_tests(testapps, root_output_dir, output_dir, FLAGS.artifact_name)
 
   _summarize_results(testapps, platforms, failures, root_output_dir, FLAGS.artifact_name)
   return 1 if failures else 0
@@ -329,7 +330,7 @@ def _build(
   return failures
 
 
-def _collect_integration_tests(testapps, output_dir, artifact_name):
+def _collect_integration_tests(testapps, root_output_dir, output_dir, artifact_name):
   testapps_artifact_dir = "testapps-" + artifact_name
   android_testapp_extension = ".apk"
   ios_testapp_extension = ".ipa"
@@ -349,7 +350,8 @@ def _collect_integration_tests(testapps, output_dir, artifact_name):
           or file_name.endswith(ios_testapp_extension)):
         testapp_paths.append(os.path.join(file_dir, file_name))
 
-  artifact_path = os.path.join(output_dir, testapps_artifact_dir)
+  artifact_path = os.path.join(root_output_dir, testapps_artifact_dir)
+  _rm_dir_safe(artifact_path)
   for testapp in testapps:
     os.makedirs(os.path.join(artifact_path, testapp))
   for path in testapp_paths:
@@ -362,7 +364,7 @@ def _collect_integration_tests(testapps, output_dir, artifact_name):
         break
 
 
-def _summarize_results(testapps, platforms, failures, output_dir, artifact_name):
+def _summarize_results(testapps, platforms, failures, root_output_dir, artifact_name):
   """Logs a readable summary of the results of the build."""
   file_name = "build-results-" + artifact_name + ".log"
 
@@ -374,13 +376,20 @@ def _summarize_results(testapps, platforms, failures, output_dir, artifact_name)
   if not failures:
     summary.append("ALL BUILDS SUCCEEDED")
   else:
-    summary.append("SOME FAILURES OCCURRED:")
+    summary.append("SOME ERRORS OCCURRED:")
     for i, failure in enumerate(failures, start=1):
       summary.append("%d: %s" % (i, failure.describe()))
   summary = "\n".join(summary)
 
   logging.info(summary)
-  test_validation.write_summary(output_dir, summary, file_name=file_name)
+  test_validation.write_summary(root_output_dir, summary, file_name=file_name)
+
+  summary_json = {}
+  summary_json["type"] = "build"
+  summary_json["testapps"] = testapps
+  summary_json["errors"] = {failure.testapp:failure.error_message for failure in failures}
+  with open(os.path.join(root_output_dir, file_name+".json"), "a") as f:
+    f.write(json.dumps(summary_json, indent=2))
 
 
 def _build_desktop(sdk_dir, cmake_flags):
@@ -465,10 +474,11 @@ def _generate_makefiles_from_repo(repo_dir):
   """Generates cmake makefiles for building iOS frameworks from SDK source."""
   ios_framework_builder = os.path.join(
       repo_dir, "build_scripts", "ios", "build.sh")
-
+  output_path = os.path.join(repo_dir, "ios_build")
+  _rm_dir_safe(output_path)
   framework_builder_args = [
       ios_framework_builder,
-      "-b", os.path.join(repo_dir, "ios_build"),
+      "-b", output_path,
       "-s", repo_dir,
       "-c", "false"
   ]
@@ -508,7 +518,7 @@ def _build_ios(
   if not ios_framework_exist:
     _build_ios_framework_from_repo(repo_dir, api_config)
     sdk_dir = os.path.join(repo_dir, "ios_build")
-    logging.info("iOS xcframework created at: %s", " ".join(sdk_dir))
+    logging.info("iOS xcframework created at: %s", sdk_dir)
 
   build_dir = os.path.join(project_dir, "ios_build")
   os.makedirs(build_dir)