Add retries to the FTL integration test runs (#876)

* Add retries to the FTL integration test runs

* Fix ModuleNotFound problems

* Make Analytics intentionally fail

* Revert "Make Analytics intentionally fail"

This reverts commit 6969172.

Author: a-maurice

.github/workflows/integration_tests.yml

@@ -526,6 +526,8 @@ jobs:
           testapp_dir: testapps
           test_type: "game-loop"
           test_devices: ${{ matrix.device_detail }}
+          max_attempts: 3
+          validator: ${GITHUB_WORKSPACE}/scripts/gha/read_ftl_test_result.py
       - name: Read FTL Test Result
         if: ${{ matrix.device_type == 'real' && !cancelled() }}
         timeout-minutes: 60

scripts/gha/__init__.py

@@ -0,0 +1 @@
+# Copyright 2023 Google

scripts/gha/integration_testing/__init__.py

@@ -0,0 +1 @@
+# Copyright 2023 Google

scripts/gha/read_ftl_test_result.py

@@ -32,11 +32,13 @@
 import attr
 import subprocess
 import json
+import sys
 
 from absl import app
 from absl import flags
 from absl import logging
 
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
 from integration_testing import test_validation
 from integration_testing import gcs
 
@@ -59,18 +61,7 @@ def main(argv):
   test_result = json.loads(FLAGS.test_result)
   tests = []
   for app in test_result.get("apps"):
-    app_path = app.get("testapp_path")
-    return_code = app.get("return_code")
-    logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
-    if return_code == 0:
-      gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
-      logging.info("gcs_dir: %s" % gcs_dir)
-      logs = _get_testapp_log_text_from_gcs(gcs_dir)
-      logging.info("Test result: %s", logs)
-      tests.append(Test(testapp_path=app_path, logs=logs))
-    else:
-      logging.error("Test failed: %s", app)
-      tests.append(Test(testapp_path=app_path, logs=None))
+    tests.append(_parse_testapp_to_test(app))
 
   (output_dir, file_name) = os.path.split(os.path.abspath(FLAGS.output_path))
   return test_validation.summarize_test_results(
@@ -80,6 +71,22 @@ def main(argv):
     file_name=file_name)
 
 
+def _parse_testapp_to_test(app):
+  """Parsed the given testapp run into a Test object."""
+  app_path = app.get("testapp_path")
+  return_code = app.get("return_code")
+  logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
+  if return_code == 0:
+    gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
+    logging.info("gcs_dir: %s" % gcs_dir)
+    logs = _get_testapp_log_text_from_gcs(gcs_dir)
+    logging.info("Test result: %s", logs)
+    return Test(testapp_path=app_path, logs=logs)
+  else:
+    logging.error("Test failed: %s", app)
+    return Test(testapp_path=app_path, logs=None)
+
+
 def _get_testapp_log_text_from_gcs(gcs_path):
   """Gets the testapp log text generated by game loops."""
   try:
@@ -130,6 +137,18 @@ def _gcs_read_file(gcs_path):
   return result.stdout
 
 
+
+def validate(test_summary):
+  """Validates a given test summary, assuming it is from a Unity testapp."""
+  if not test_summary:
+    logging.error("Nothing to be validate! Please provide test_summary")
+    return False
+  
+  test = _parse_testapp_to_test(test_summary)
+  result = test_validation.validate_results(test.logs, "unity")
+  return result.complete and result.fails == 0
+
+
 if __name__ == '__main__':
   flags.mark_flag_as_required("test_result")
   flags.mark_flag_as_required("output_path")