Merge pull request #197 from firebase/feature/aks-improve-logging

Improve logging in integration test workflows

Author: anonymous-akorn

.github/workflows/cpp-packaging.yml

@@ -677,3 +677,7 @@ jobs:
       if: matrix.target_platform != 'Desktop' && !cancelled()
       run: |
         python scripts/gha/test_lab.py --android_model ${{ env.android_device }} --android_api ${{ env.android_api }} --ios_model ${{ env.ios_device }} --ios_version ${{ env.ios_version }} --testapp_dir testapps --code_platform cpp --key_file scripts/gha-encrypted/gcs_key_file.json
+    - name: Summarize build and test results
+      if: ${{ !cancelled() }}
+      shell: bash
+      run: cat testapps/summary.log

.github/workflows/integration_tests.yml

@@ -116,10 +116,14 @@ jobs:
           pip install -r scripts/gha/requirements.txt
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
 
-      - name: Build integration tests (and run Desktop tests)
+      - name: Build integration tests
         run: |
-          python scripts/gha/build_testapps.py --t ${{ github.event.inputs.apis }} --p ${{ matrix.target_platform }} --output_directory ${{ github.workspace }} --use_vcpkg --execute_desktop_testapp --noadd_timestamp
+          python scripts/gha/build_testapps.py --t ${{ github.event.inputs.apis }} --p ${{ matrix.target_platform }} --output_directory ${{ github.workspace }} --use_vcpkg --noadd_timestamp
 
+      - name: Run desktop integration tests
+        if: matrix.target_platform == 'Desktop' && !cancelled()
+        run: |
+          python scripts/gha/desktop_tester.py --testapp_dir testapps
       # Workaround for https://github.com/GoogleCloudPlatform/github-actions/issues/100
       # Must be run after the Python setup action
       - name: Set CLOUDSDK_PYTHON (Windows)
@@ -137,3 +141,7 @@ jobs:
         if: matrix.target_platform != 'Desktop' && !cancelled()
         run: |
           python scripts/gha/test_lab.py --android_model ${{ github.event.inputs.android_device }} --android_api ${{ github.event.inputs.android_api }} --ios_model ${{ github.event.inputs.ios_device }} --ios_version ${{ github.event.inputs.ios_version }} --testapp_dir ${{ github.workspace }}/testapps --code_platform cpp --key_file ${{ github.workspace }}/scripts/gha-encrypted/gcs_key_file.json
+      - name: Summarize build and test results
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: cat testapps/summary.log

scripts/gha/build_testapps.py

@@ -81,6 +81,7 @@
 import attr
 
 from integration_testing import config_reader
+from integration_testing import test_validation
 from integration_testing import xcodebuild
 import utils
 
@@ -145,11 +146,6 @@
     " the local spec repos available on this machine. Must also include iOS"
     " in platforms flag.")
 
-flags.DEFINE_bool(
-    "execute_desktop_testapp", True,
-    "(Desktop only) Run the testapp after building it. Will return non-zero"
-    " code if any tests fail inside the testapp.")
-
 flags.DEFINE_string(
     "compiler", None,
     "(Desktop only) Specify the compiler with CMake during the testapps build."
@@ -187,6 +183,7 @@ def main(argv):
     timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S")
   else:
     timestamp = ""
+  output_dir = os.path.join(output_dir, "testapps" + timestamp)
 
   ios_framework_dir = os.path.join(sdk_dir, "frameworks")
   ios_framework_exist = os.path.isdir(ios_framework_dir)
@@ -218,25 +215,22 @@ def main(argv):
         output_dir=output_dir,
         sdk_dir=sdk_dir,
         ios_framework_exist=ios_framework_exist,
-        timestamp=timestamp,
         root_dir=root_dir,
         ios_sdk=FLAGS.ios_sdk,
-        cmake_flags=cmake_flags,
-        execute_desktop_testapp=FLAGS.execute_desktop_testapp)
+        cmake_flags=cmake_flags)
     logging.info("END building for %s", testapp)
 
-  _summarize_results(testapps, platforms, failures)
+  _summarize_results(testapps, platforms, failures, output_dir)
   return 1 if failures else 0
 
 
 def _build(
     testapp, platforms, api_config, output_dir, sdk_dir, ios_framework_exist,
-    timestamp, root_dir, ios_sdk, cmake_flags, execute_desktop_testapp):
+    root_dir, ios_sdk, cmake_flags):
   """Builds one testapp on each of the specified platforms."""
   testapp_dir = os.path.join(root_dir, api_config.testapp_path)
   project_dir = os.path.join(
-      output_dir, "testapps" + timestamp, api_config.full_name,
-      os.path.basename(testapp_dir))
+      output_dir, api_config.full_name, os.path.basename(testapp_dir))
 
   logging.info("Copying testapp project to %s", project_dir)
   os.makedirs(project_dir)
@@ -253,8 +247,6 @@ def _build(
     logging.info("BEGIN %s, %s", testapp, _DESKTOP)
     try:
       _build_desktop(sdk_dir, cmake_flags)
-      if execute_desktop_testapp:
-        _execute_desktop_testapp(project_dir)
     except subprocess.SubprocessError as e:
       failures.append(
           Failure(testapp=testapp, platform=_DESKTOP, error_message=str(e)))
@@ -291,36 +283,30 @@ def _build(
   return failures
 
 
-def _summarize_results(testapps, platforms, failures):
+def _summarize_results(testapps, platforms, failures, output_dir):
   """Logs a readable summary of the results of the build."""
-  logging.info(
-      "FINISHED BUILDING TESTAPPS.\n\n\n"
-      "Tried to build these testapps: %s\n"
-      "On these platforms: %s",
-      ", ".join(testapps), ", ".join(platforms))
+  summary = []
+  summary.append("BUILD SUMMARY:")
+  summary.append("TRIED TO BUILD: " + ",".join(testapps))
+  summary.append("ON PLATFORMS: " + ",".join(platforms))
+
   if not failures:
-    logging.info("No failures occurred")
+    summary.append("ALL BUILDS SUCCEEDED")
   else:
-    # Collect lines, then log once, to reduce logging noise from timestamps etc.
-    lines = ["Some failures occurred:"]
+    summary.append("SOME FAILURES OCCURRED:")
     for i, failure in enumerate(failures, start=1):
-      lines.append("%d: %s" % (i, failure.describe()))
-    logging.info("\n".join(lines))
+      summary.append("%d: %s" % (i, failure.describe()))
+  summary = "\n".join(summary)
+
+  logging.info(summary)
+  test_validation.write_summary(output_dir, summary)
 
 
 def _build_desktop(sdk_dir, cmake_flags):
   _run(["cmake", ".", "-DFIREBASE_CPP_SDK_DIR=" + sdk_dir] + cmake_flags)
   _run(["cmake", "--build", "."])
 
 
-def _execute_desktop_testapp(project_dir):
-  if platform.system() == "Windows":
-    testapp_path = os.path.join(project_dir, "Debug", "integration_test.exe")
-  else:
-    testapp_path = os.path.join(project_dir, "integration_test")
-  _run([testapp_path], timeout=300)
-
-
 def _get_desktop_compiler_flags(compiler, compiler_table):
   """Returns the command line flags for this compiler."""
   if not compiler:  # None is an acceptable default value
@@ -461,14 +447,10 @@ def _build_ios(
 
   podfile_tool_path = os.path.join(
       root_dir, "scripts", "gha", "integration_testing", "update_podfile.py")
-  sdk_podfile_path = os.path.join(
-      root_dir, "ios_pod", "Podfile")
-  app_podfile_path = os.path.join(
-      project_dir, "Podfile")
   podfile_patcher_args = [
       sys.executable, podfile_tool_path,
-      "--sdk_podfile", sdk_podfile_path,
-      "--app_podfile", app_podfile_path
+      "--sdk_podfile", os.path.join(root_dir, "ios_pod", "Podfile"),
+      "--app_podfile", os.path.join(project_dir, "Podfile")
   ]
   _run(podfile_patcher_args)
   _run(["pod", "install"])

scripts/gha/desktop_tester.py

@@ -0,0 +1,112 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runs and validates desktop C++ testapps.
+
+Usage:
+  python desktop_tester.py --testapp_dir ~/Downloads/testapps
+
+This will search --testapp_dir for files whose name is given by --testapp_name
+(default: "integration_test", or "integration_test.exe" on Windows), execute
+them, parse their log output according to the C++ test summary format, and
+report a summary of results.
+
+"""
+
+import os
+import platform
+import subprocess
+import threading
+
+from absl import app
+from absl import flags
+from absl import logging
+import attr
+
+from integration_testing import test_validation
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string("testapp_dir", None, "Look for testapps in this directory.")
+flags.DEFINE_string("testapp_name", "integration_test", "Name of the testapps.")
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError("Too many command-line arguments.")
+
+  testapp_dir = _fix_path(FLAGS.testapp_dir)
+
+  testapp_name = FLAGS.testapp_name
+  if platform.system() == "Windows":
+    testapp_name += ".exe"
+
+  testapps = []
+  for file_dir, _, file_names in os.walk(testapp_dir):
+    for file_name in file_names:
+      if file_name == testapp_name:
+        testapps.append(os.path.join(file_dir, file_name))
+  if not testapps:
+    logging.error("No testapps found.")
+    test_validation.write_summary(testapp_dir, "Desktop tests: none found.")
+    return 1
+  logging.info("Testapps found: %s\n", "\n".join(testapps))
+
+  tests = [Test(testapp_path=testapp) for testapp in testapps]
+
+  logging.info("Running tests...")
+  threads = []
+  for test in tests:
+    thread = threading.Thread(target=test.run)
+    threads.append(thread)
+    thread.start()
+  for thread in threads:
+    thread.join()
+
+  return test_validation.summarize_test_results(
+      tests, test_validation.CPP, testapp_dir)
+
+
+def _fix_path(path):
+  """Expands ~, normalizes slashes, and converts relative paths to absolute."""
+  return os.path.abspath(os.path.expanduser(path))
+
+
+@attr.s(frozen=False, eq=False)
+class Test(object):
+  """Holds data related to the testing of one testapp."""
+  testapp_path = attr.ib()
+  # This will be populated after the test completes, instead of initialization.
+  logs = attr.ib(init=False, default=None)
+
+  # This runs in a separate thread, so instead of returning values we store
+  # them as fields so they can be accessed from the main thread.
+  def run(self):
+    """Executes this testapp."""
+    result = subprocess.run(
+        args=[self.testapp_path],
+        cwd=os.path.dirname(self.testapp_path),  # Testapp checks CWD for config
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=False,
+        timeout=300)
+    logging.info("Finished running %s", self.testapp_path)
+
+    self.logs = result.stdout
+    self.return_code = result.returncode
+
+
+if __name__ == "__main__":
+  app.run(main)

scripts/gha/integration_testing/gcs.py

@@ -89,6 +89,17 @@ def authorize_gcs(key_file):
       check=True)
 
 
+# This is intended to be logged when a tool stores artifacts on GCS.
+def get_gsutil_tips():
+  """Returns a human readable string with tips on accessing a GCS bucket."""
+  return "\n".join((
+      "GCS Advice: Install the Google Cloud SDK to access the gsutil tool.",
+      "Use 'gsutil ls <gs_uri>' to list contents of a directory on GCS.",
+      "Use 'gsutil cp <gs_uri> <local_path>' to copy an artifact.",
+      "Use 'gsutil -m cp -r <gs_uri> <local_path>' to copy a directory."
+  ))
+
+
 def _verify_gcloud_sdk_command_line_tools():
   """Verifies the presence of the gCloud SDK's command line tools."""
   logging.info("Looking for gcloud and gsutil tools...")