Create build matrices & test matrices for integration test workflow (#415)

Author: sunmou99

.github/workflows/integration_tests.yml

@@ -106,8 +106,8 @@
 _SUPPORTED_PLATFORMS = (_ANDROID, _IOS, _DESKTOP)
 
 # Values for iOS SDK flag (where the iOS app will run)
-_IOS_SDK_DEVICE = "device"
-_IOS_SDK_SIMULATOR = "simulator"
+_IOS_SDK_DEVICE = "real"
+_IOS_SDK_SIMULATOR = "virtual"
 _SUPPORTED_IOS_SDK = (_IOS_SDK_DEVICE, _IOS_SDK_SIMULATOR)
 
 FLAGS = flags.FLAGS
@@ -120,6 +120,12 @@
     "output_directory", "~",
     "Build output will be placed in this directory.")
 
+flags.DEFINE_string(
+    "artifact_name", "",
+    "artifacts will be created and placed in output_directory."
+    " testapps artifact is testapps-$artifact_name;"
+    " build log artifact is build-results-$artifact_name.log.")    
+
 flags.DEFINE_string(
     "repo_dir", os.getcwd(),
     "Firebase C++ SDK Git repository. Current directory by default.")
@@ -141,8 +147,8 @@
 
 flags.DEFINE_list(
     "ios_sdk", _IOS_SDK_DEVICE, 
-    "(iOS only) Build for device (ipa), simulator (app), or both."
-    " Building for both will produce both an .app and an .ipa.")
+    "(iOS only) Build for real device (.ipa), virtual device / simulator (.app), "
+    "or both. Building for both will produce both an .app and an .ipa.")
 
 flags.DEFINE_bool(
     "update_pod_repo", True,
@@ -186,7 +192,7 @@ def main(argv):
   testapps = FLAGS.testapps
 
   sdk_dir = _fix_path(FLAGS.packaged_sdk or FLAGS.repo_dir)
-  output_dir = _fix_path(FLAGS.output_directory)
+  root_output_dir = _fix_path(FLAGS.output_directory)
   repo_dir = _fix_path(FLAGS.repo_dir)
 
   update_pod_repo = FLAGS.update_pod_repo
@@ -196,9 +202,9 @@ def main(argv):
     timestamp = ""
 
   if FLAGS.short_output_paths:
-    output_dir = os.path.join(output_dir, "ta")
+    output_dir = os.path.join(root_output_dir, "ta")
   else:
-    output_dir = os.path.join(output_dir, "testapps" + timestamp)
+    output_dir = os.path.join(root_output_dir, "testapps" + timestamp)
 
   ios_framework_dir = os.path.join(sdk_dir, "xcframeworks")
   ios_framework_exist = os.path.isdir(ios_framework_dir)
@@ -248,8 +254,10 @@ def main(argv):
           cmake_flags=cmake_flags,
           short_output_paths=FLAGS.short_output_paths)
       logging.info("END building for %s", testapp)
+  
+  _collect_integration_tests(testapps, root_output_dir, FLAGS.artifact_name)
 
-  _summarize_results(testapps, platforms, failures, output_dir)
+  _summarize_results(testapps, platforms, failures, root_output_dir, FLAGS.artifact_name)
   return 1 if failures else 0
 
 
@@ -321,8 +329,43 @@ def _build(
   return failures
 
 
-def _summarize_results(testapps, platforms, failures, output_dir):
+def _collect_integration_tests(testapps, output_dir, artifact_name):
+  testapps_artifact_dir = "testapps-" + artifact_name
+  android_testapp_extension = ".apk"
+  ios_testapp_extension = ".ipa"
+  ios_simualtor_testapp_extension = ".app"
+  desktop_testapp_name = "integration_test" 
+  if platform.system() == "Windows":
+    desktop_testapp_name += ".exe"
+
+  testapp_paths = []
+  for file_dir, directories, file_names in os.walk(output_dir):
+    for directory in directories:
+      if directory.endswith(ios_simualtor_testapp_extension):
+        testapp_paths.append(os.path.join(file_dir, directory))
+    for file_name in file_names:
+      if ((file_name == desktop_testapp_name and "ios_build" not in file_dir) 
+          or file_name.endswith(android_testapp_extension) 
+          or file_name.endswith(ios_testapp_extension)):
+        testapp_paths.append(os.path.join(file_dir, file_name))
+
+  artifact_path = os.path.join(output_dir, testapps_artifact_dir)
+  for testapp in testapps:
+    os.makedirs(os.path.join(artifact_path, testapp))
+  for path in testapp_paths:
+    for testapp in testapps:
+      if testapp in path:
+        if os.path.isfile(path):
+          shutil.copy(path, os.path.join(artifact_path, testapp))
+        else:
+          dir_util.copy_tree(path, os.path.join(artifact_path, testapp, "integration_test.app"))
+        break
+
+
+def _summarize_results(testapps, platforms, failures, output_dir, artifact_name):
   """Logs a readable summary of the results of the build."""
+  file_name = "build-results-" + artifact_name + ".log"
+
   summary = []
   summary.append("BUILD SUMMARY:")
   summary.append("TRIED TO BUILD: " + ",".join(testapps))
@@ -337,7 +380,7 @@ def _summarize_results(testapps, platforms, failures, output_dir):
   summary = "\n".join(summary)
 
   logging.info(summary)
-  test_validation.write_summary(output_dir, summary)
+  test_validation.write_summary(output_dir, summary, file_name=file_name)
 
 
 def _build_desktop(sdk_dir, cmake_flags):

scripts/gha/build_testapps.py

@@ -40,6 +40,10 @@
 
 flags.DEFINE_string("testapp_dir", None, "Look for testapps in this directory.")
 flags.DEFINE_string("testapp_name", "integration_test", "Name of the testapps.")
+flags.DEFINE_string(
+    "logfile_name", "",
+    "Create test log artifact test-results-$logfile_name.log."
+    " logfile will be created and placed in testapp_dir.")  
 
 
 def main(argv):
@@ -55,7 +59,8 @@ def main(argv):
   testapps = []
   for file_dir, _, file_names in os.walk(testapp_dir):
     for file_name in file_names:
-      if file_name == testapp_name:
+      # ios build create intermediates file with same name, filter them out 
+      if file_name == testapp_name and "ios_build" not in file_dir:
         testapps.append(os.path.join(file_dir, file_name))
   if not testapps:
     logging.error("No testapps found.")
@@ -75,7 +80,10 @@ def main(argv):
     thread.join()
 
   return test_validation.summarize_test_results(
-      tests, test_validation.CPP, testapp_dir)
+      tests, 
+      test_validation.CPP, 
+      testapp_dir, 
+      file_name="test-results-" + FLAGS.logfile_name + ".log")
 
 
 def _fix_path(path):
@@ -95,6 +103,7 @@ class Test(object):
   def run(self):
     """Executes this testapp."""
     result = None  # Ensures this var is defined if timeout occurs.
+    os.chmod(self.testapp_path, 0o777)
     try:
       result = subprocess.run(
           args=[self.testapp_path],

scripts/gha/desktop_tester.py

@@ -294,7 +294,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				ONLY_ACTIVE_ARCH = YES;
@@ -349,7 +349,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				MTL_FAST_MATH = YES;
 				SDKROOT = iphoneos;
@@ -366,7 +366,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CODE_SIGN_STYLE = Automatic;
 				INFOPLIST_FILE = gameloop/Info.plist;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",
@@ -385,7 +385,7 @@
 				ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
 				CODE_SIGN_STYLE = Automatic;
 				INFOPLIST_FILE = gameloop/Info.plist;
-				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
 				LD_RUNPATH_SEARCH_PATHS = (
 					"$(inherited)",
 					"@executable_path/Frameworks",

scripts/gha/integration_testing/gameloop/gameloop.xcodeproj/project.pbxproj

@@ -31,7 +31,7 @@ class MainActivity : AppCompatActivity(R.layout.activity_main) {
     val pkgAppsList: List<ResolveInfo> = packageManager.queryIntentActivities(intent, 0)
     val gamePackageName = pkgAppsList[0].activityInfo.packageName
 
-    val dir = File(getExternalFilesDir(null), gamePackageName)
+    val dir = File(getFilesDir(), gamePackageName)
     if (!dir.exists()) dir.mkdirs()
     val filename = "Results1.json"
     val file = File(dir, filename)

scripts/gha/integration_testing/gameloop_android/app/src/main/java/com/google/firebase/gameloop/MainActivity.kt

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <paths>
-    <external-path
+    <files-path
         name="files"
         path="/"/>
 </paths>

scripts/gha/integration_testing/gameloop_android/app/src/main/res/xml/file_paths.xml

@@ -124,7 +124,7 @@ def validate_results_cpp(log_text):
       summary=result_summary)
 
 
-def summarize_test_results(tests, platform, summary_dir, extra_info=""):
+def summarize_test_results(tests, platform, summary_dir, file_name="summary.log", extra_info=""):
   """Summarizes and logs test results for multiple tests.
 
   Each 'test' should be an object with properties "testapp_path", which
@@ -199,12 +199,12 @@ def summarize_test_results(tests, platform, summary_dir, extra_info=""):
       % (len(tests), len(successes), len(failures), len(errors)))
   summary = "\n".join(summary)
   logging.info(summary)
-  write_summary(summary_dir, summary)
+  write_summary(summary_dir, summary, file_name)
 
   return 0 if len(tests) == len(successes) else 1
 
 
-def write_summary(testapp_dir, summary):
+def write_summary(testapp_dir, summary, file_name="summary.log"):
   """Writes a summary of tests/builds to a file in the testapp directory.
 
   This will append the given summary to a file in the testapp directory,
@@ -217,7 +217,7 @@ def write_summary(testapp_dir, summary):
 
   """
   # This method serves as the source of truth on where to put the summary.
-  with open(os.path.join(testapp_dir, "summary.log"), "a") as f:
+  with open(os.path.join(testapp_dir, file_name), "a") as f:
     # The timestamp mainly helps when running locally: if running multiple
     # tests on the same directory, the results will accumulate, with a timestamp
     # to help keep track of when a given test was run.

scripts/gha/integration_testing/test_validation.py

@@ -45,7 +45,7 @@ def get_args_for_build(
     scheme (str): Name of the scheme to build.
     output_dir (str): Directory for the resulting build artifacts. Will be
         created if it doesn't already exist.
-    ios_sdk (str): Where this build will be run: device or simulator.
+    ios_sdk (str): Where this build will be run: real device or virtual device (simulator).
     configuration (str): Value for the -configuration flag.
 
   Returns:
@@ -77,16 +77,16 @@ def get_args_for_build(
 
 def _get_ios_env_from_target(ios_sdk):
   """Return a value for the -sdk flag based on the target (device/simulator)."""
-  if ios_sdk == "device":
+  if ios_sdk == "real":
     return "iphoneos"
-  elif ios_sdk == "simulator":
+  elif ios_sdk == "virtual":
     return "iphonesimulator"
   else:
     raise ValueError("Unrecognized ios_sdk: %s" % ios_sdk)
 
 
 def generate_unsigned_ipa(output_dir, configuration):
-  """create unsigned .ipa from .app
+  """create unsigned .ipa from .app, then remove .app afterwards
 
   Args:
     output_dir (str): Same value as get_args_for_build. generated unsigned .ipa 
@@ -101,3 +101,4 @@ def generate_unsigned_ipa(output_dir, configuration):
   shutil.move(app_path, payload_path)
   shutil.make_archive(payload_path, 'zip', root_dir=iphone_build_dir, base_dir='Payload')
   shutil.move('%s.%s'%(payload_path, 'zip'), ipa_path)
+  shutil.rmtree(payload_path)

scripts/gha/integration_testing/xcodebuild.py

@@ -99,15 +99,27 @@
     "matrix": {
       "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
       "platform": ["Desktop", "Android", "iOS"],
-      "ssl_lib": ["openssl", "boringssl"]
+      "ssl_lib": ["openssl", "boringssl"],
+      "android_device": ["virtual:system-images;android-28;google_apis;x86_64+28.0.3", "real:flame+29"],
+      "ios_device": ["virtual:iPhone 8+12.0", "real:iphone8+12.4"],
+      EXPANDED_KEY: {
+        "android_device": ["virtual:system-images;android-18;default;x86+28.0.3", 
+                          "virtual:system-images;android-28;google_apis;x86_64+28.0.3", 
+                          "virtual:system-images;android-29;default;x86+28.0.3", 
+                          "real:Nexus10+19", 
+                          "real:Pixel2+28", 
+                          "real:flame+29"],
+        "ios_device": ["virtual:iPhone 6+11.4", 
+                      "virtual:iPhone 8+12.0", 
+                      "virtual:iPhone 11+14.4", 
+                      "real:iphone8+11.4", 
+                      "real:iphone6s+12.0", 
+                      "real:iphone11pro+14.1"]
+      }
     },
     "config": {
       "apis": "admob,analytics,auth,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage",
-      "mobile_test_on": "device,simulator",
-      "android_device": "flame",
-      "android_api": "29",
-      "ios_device": "iphone8",
-      "ios_version": "12.4"
+      "mobile_test_on": "real"
     }
   },
 
@@ -172,6 +184,18 @@ def get_value(workflow, use_expanded, parm_key, config_parms_only=False):
                                                                 use_expanded))
 
 
+def filter_value(workflow, parm_key, value, by):
+  """ Filter value by condtional
+
+  Currently only used for filter mobile device by device type.
+  """
+  if workflow == "integration_tests" and (parm_key == "android_device" or parm_key == "ios_device"):
+    filtered_value = filter(lambda device: device.split(":")[0] in by, value)
+    value = list(filtered_value)
+
+  return value  
+
+
 def print_value(value):
   """ Print Json formatted string that can be consumed in Github workflow."""
   # Eg: for lists,
@@ -276,6 +300,7 @@ def main():
     return
 
   value = get_value(args.workflow, args.expanded, args.parm_key, args.config)
+  value = filter_value(args.workflow, args.parm_key, value, by = args.device)
   if args.auto_diff:
     value = filter_values_on_diff(args.parm_key, value, args.auto_diff)
   print_value(value)
@@ -289,6 +314,7 @@ def parse_cmdline_args():
   parser.add_argument('-k', '--parm_key', required=True, help='Print the value of specified key from matrix or config maps.')
   parser.add_argument('-a', '--auto_diff', metavar='BRANCH', help='Compare with specified base branch to automatically set matrix options')
   parser.add_argument('-o', '--override', help='Override existing value with provided value')
+  parser.add_argument('-d', '--device', default=['real', 'virtual'], help='Test on which type of mobile devices')
   args = parser.parse_args()
   return args
 

scripts/gha/print_matrix_configuration.py

@@ -49,6 +49,7 @@
 flags.DEFINE_string("repo_dir", os.getcwd(), "Path to C++ SDK Github repo.")
 flags.DEFINE_string("passphrase", None, "The passphrase itself.")
 flags.DEFINE_string("passphrase_file", None, "Path to file with passphrase.")
+flags.DEFINE_string("artifact", None, "Artifact Path, google-services.json will be placed here.")
 
 
 def main(argv):
@@ -78,6 +79,13 @@ def main(argv):
       api = os.path.basename(os.path.dirname(path))
       file_name = os.path.basename(path).replace(".gpg", "")
       dest_paths = [os.path.join(repo_dir, api, "integration_test", file_name)]
+      if FLAGS.artifact:
+        # /<repo_dir>/<artifact>/auth/google-services.json
+        if "google-services" in path and os.path.isdir(os.path.join(repo_dir, FLAGS.artifact, api)):
+          dest_paths = [os.path.join(repo_dir, FLAGS.artifact, api, file_name)]
+        else:
+          continue
+
       # Some apis like Firestore also have internal integration tests.
       if os.path.exists( os.path.join(repo_dir, api, "integration_test_internal")):
         dest_paths.append(os.path.join(repo_dir, api,
@@ -93,6 +101,9 @@ def main(argv):
           _patch_reverse_id(dest_path)
           _patch_bundle_id(dest_path)
 
+  if FLAGS.artifact:
+    return
+
   print("Attempting to patch Dynamic Links uri prefix.")
   uri_path = os.path.join(secrets_dir, "dynamic_links", "uri_prefix.txt.gpg")
   uri_prefix = _decrypt(uri_path, passphrase)