GHA integration test (#637)

Add GHA for tests

Currently only run tests except for NUnit tests since it is broken from Unity 2019.
Also add logs when test starts and summary when all the tests are finished.

Author: chkuang-g

.github/workflows/test.yaml

@@ -1,19 +1,145 @@
 name: Test
 
 on:
+  schedule:
+    - cron: "0 11 * * *"  # 11am UTC = 3`am PST
+
+  pull_request:
+    types: [ labeled, closed ]
+
   workflow_dispatch:
     inputs:
-      unity_versions:
-        description: 'Unity version (value: 2018, 2019, 2020. separated by commas)'
+      unity_version:
+        description: 'Unity version (value: 2018, 2019, 2020)'
         default: '2019'
         required: true
+      include_test_types:
+        description: 'Specify the only types of tests to run, separated by comma. See TestTypesEnum in build.gradle for options.'
+        default: ''
+        required: false
+      exclude_test_types:
+        description: 'Specify the types of tests to exclude, separated by comma. See TestTypesEnum in build.gradle for options.'
+        default: ''
+        required: false
+      include_test_modules:
+        description: 'Specify the only modules to test against, separated by comma. See TestModulesEnum in build.gradle for options.'
+        default: ''
+        required: false
+      exclude_test_modules:
+        description: 'Specify the modules to exclude from testing against, separated by comma. See TestModulesEnum in build.gradle for options.'
+        default: ''
+        required: false
+      exclude_tests:
+        description: 'Specify the tests to exclude, separated by comma. See the tasks in build.gradle for options.'
+        default: ''
+        required: false
 
 env:
   pythonVersion: '3.7'
   artifactRetentionDays: 2
 
 jobs:
+  check_and_prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      unity_version: ${{ steps.set_outputs.outputs.unity_version }}
+      include_test_types: ${{ steps.set_outputs.outputs.include_test_types }}
+      exclude_test_types: ${{ steps.set_outputs.outputs.exclude_test_types }}
+      include_test_modules: ${{ steps.set_outputs.outputs.include_test_modules }}
+      exclude_test_modules: ${{ steps.set_outputs.outputs.exclude_test_modules }}
+      exclude_tests: ${{ steps.set_outputs.outputs.exclude_tests }}
+    steps:
+      - id: set_outputs
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "unity_version=${{ github.event.inputs.unity_version }}" >> $GITHUB_OUTPUT
+            echo "include_test_types=${{ github.event.inputs.include_test_types }}" >> $GITHUB_OUTPUT
+            echo "exclude_test_types=${{ github.event.inputs.exclude_test_types }}" >> $GITHUB_OUTPUT
+            echo "include_test_modules=${{ github.event.inputs.include_test_modules }}" >> $GITHUB_OUTPUT
+            echo "exclude_test_modules=${{ github.event.inputs.exclude_test_modules }}" >> $GITHUB_OUTPUT
+            echo "exclude_tests=${{ github.event.inputs.exclude_tests }}" >> $GITHUB_OUTPUT
+          else
+            # inputs are not available for non "workflow_dispatch" events. Therefore, set default value here.
+            echo "unity_version=2019" >> $GITHUB_OUTPUT
+            echo "include_test_types=" >> $GITHUB_OUTPUT
+            echo "exclude_test_types=" >> $GITHUB_OUTPUT
+            echo "include_test_modules=" >> $GITHUB_OUTPUT
+            echo "exclude_test_modules=" >> $GITHUB_OUTPUT
+            echo "exclude_tests=" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Print output
+        run: |
+          echo outputs.unity_version : ${{ steps.set_outputs.outputs.unity_version }}
+          echo outputs.include_test_types : ${{ steps.set_outputs.outputs.include_test_types }}
+          echo outputs.exclude_test_types : ${{ steps.set_outputs.outputs.exclude_test_types }}
+          echo outputs.include_test_modules : ${{ steps.set_outputs.outputs.include_test_modules }}
+          echo outputs.exclude_test_modules : ${{ steps.set_outputs.outputs.exclude_test_modules }}
+          echo outputs.exclude_tests : ${{ steps.set_outputs.outputs.exclude_tests }}
+
   test_on_macos:
+    name: test-macOS-unity${{ needs.check_and_prepare.outputs.unity_version }}
     runs-on: macos-latest
+    needs: [check_and_prepare]
+    strategy:
+      fail-fast: false
     steps:
-    - run: echo "Hello World"
+      - uses: actions/checkout@v3
+      - id: build_setup
+        uses: ./gha/build_setup
+        timeout-minutes: 30
+        with:
+          unity_version: ${{ needs.check_and_prepare.outputs.unity_version }}
+          platform: macOS
+          python_version: ${{ env.pythonVersion }}
+          unity_username: ${{ secrets.UNITY_USERNAME }}
+          unity_password: ${{ secrets.UNITY_PASSWORD }}
+          unity_serial_id: ${{ secrets.SERIAL_ID }}
+
+      - name: Set Unity Env for EDM4U build script
+        shell: bash
+        run: echo "UNITY_EXE=${{ env.UNITY_ROOT_DIR }}/Unity.app/Contents/MacOS/Unity" >> $GITHUB_ENV
+
+      - name: Run tests
+        shell: bash
+        timeout-minutes: 60
+        run: |
+          # DISABLE: NUnit test is currently broken from Unity 2019
+          # DISABLE: testVersionHandlerActivationBatchMode is causing timeout.
+          ./gradlew test -q \
+                    -PINTERACTIVE_MODE_TESTS_ENABLED=0 \
+                    -PINCLUDE_TEST_TYPES="${{ needs.check_and_prepare.outputs.include_test_types }}" \
+                    -PEXCLUDE_TEST_TYPES="NUnit,${{ needs.check_and_prepare.outputs.exclude_test_types }}" \
+                    -PINCLUDE_TEST_MODULES="${{ needs.check_and_prepare.outputs.include_test_modules }}" \
+                    -PEXCLUDE_TEST_MODULES="${{ needs.check_and_prepare.outputs.exclude_test_modules }}" \
+                    -PEXCLUDE_TESTS="testVersionHandlerActivationBatchMode,${{ needs.check_and_prepare.outputs.exclude_tests }}"
+
+      - name: Print test log
+        if: always()
+        shell: bash
+        continue-on-error: true
+        run: cat test_output/test*IntegrationTestsBatchMode/*.log
+
+      - name: Obtain Failed tests
+        if: always()
+        shell: bash
+        continue-on-error: true
+        run: |
+          cat test_output/test*/*_test.log | grep "^Test .* PASSED$"
+          cat test_output/test*/*_test.log | grep "^Test .* FAILED$"
+          cat test_output/test*/*_test.log | grep "^Test .* SKIPPED$"
+
+      - name: Return Unity license
+        if: always()
+        uses: firebase/firebase-unity-sdk/gha/unity@main
+        with:
+          version: ${{ inputs.unity_version }}
+          release_license: "true"
+
+      - name: Upload build logs
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: logs
+          path: test_output/test*/*.log
+          retention-days: ${{ env.artifactRetentionDays }}

build.gradle

@@ -526,6 +526,9 @@ void setTestProperties(Task task, TestTypeEnum type, TestModuleEnum module) {
  */
 task(testTestTypeEnum) { task ->
   setTestProperties(task, TestTypeEnum.GRADLE, TestModuleEnum.TOOL)
+  doFirst {
+    ReportTestStarted(task)
+  }
   doLast {
     def expectedTestTypeEnumCompleteSet =
       new HashSet([
@@ -570,6 +573,9 @@ task(testTestTypeEnum) { task ->
  */
 task(testTestModuleEnum) { task ->
   setTestProperties(task, TestTypeEnum.GRADLE, TestModuleEnum.TOOL)
+  doFirst {
+    ReportTestStarted(task)
+  }
   doLast {
     def expectedTestModuleEnumCompleteSet =
       new HashSet([
@@ -1480,6 +1486,9 @@ Task createUnityTestTask(String taskName, String description,
     doLast {
       EvaluateTestResult(task)
     }
+    doFirst {
+      ReportTestStarted(task)
+    }
     setTestProperties(task, testType, testModule)
   }
 
@@ -1713,54 +1722,62 @@ task testDownloadArtifacts(type: GradleBuild) { task ->
   doLast {
     EvaluateTestResult(task)
   }
+  doFirst {
+    ReportTestStarted(task)
+  }
   finalizedBy "reportAllTestsResult"
 }
 
+
+/*
+ * Report when a test starts to run
+ *
+ * @param testTask Task for test to start.
+ */
+void ReportTestStarted(Task testTask) {
+  println sprintf("Test %s STARTED", testTask.name)
+}
+
 /*
  * Evaluate previously-ran test result
  *
  * @param testTask Task for previously-ran test
  */
 void EvaluateTestResult(Task testTask) {
+  Boolean succeeded = false
   if (testTask.class.simpleName.startsWith("Exec")) {
-    if (testTask.execResult.exitValue != 0) {
-      String errorMsg = sprintf("Test %s FAILED", testTask.name)
-      println sprintf("::error::%s", errorMsg)
-      project.ext.testSessions.add(new TestSession(
-        name: testTask.name,
-        type: testTask.ext.testType,
-        module: testTask.ext.testModule,
-        isPassed: false))
-      if (!project.ext.continueOnFailForTestsEnabled) {
-        throw new GradleException(errorMsg)
-      }
-    } else {
-      println sprintf("::debug::Test %s PASSED", testTask.name, testTask.execResult.exitValue)
-      project.ext.testSessions.add(new TestSession(
-        name: testTask.name,
-        type: testTask.ext.testType,
-        module: testTask.ext.testModule,
-        isPassed: true))
+    if (testTask.execResult.exitValue == 0) {
+      succeeded = true
     }
   } else if (testTask.class.simpleName.startsWith("DefaultTask") ||
              testTask.class.simpleName.startsWith("GradleBuild")) {
     if (testTask.state.didWork && testTask.state.failure == null) {
-      project.ext.testSessions.add(new TestSession(
-        name: testTask.name,
-        type: testTask.ext.testType,
-        module: testTask.ext.testModule,
-        isPassed: true))
-    } else {
-      project.ext.testSessions.add(new TestSession(
-        name: testTask.name,
-        type: testTask.ext.testType,
-        module: testTask.ext.testModule,
-        isPassed: false))
+      succeeded = true
     }
   } else {
     throw new GradleException(
         sprintf("Unsupported test class %s", testTask.class.simpleName))
   }
+
+  if (succeeded) {
+    println sprintf("Test %s PASSED", testTask.name)
+    project.ext.testSessions.add(new TestSession(
+      name: testTask.name,
+      type: testTask.ext.testType,
+      module: testTask.ext.testModule,
+      isPassed: true))
+  } else {
+    String errorMsg = sprintf("Test %s FAILED", testTask.name)
+    println sprintf("::error::%s", errorMsg)
+    project.ext.testSessions.add(new TestSession(
+      name: testTask.name,
+      type: testTask.ext.testType,
+      module: testTask.ext.testModule,
+      isPassed: false))
+    if (!project.ext.continueOnFailForTestsEnabled) {
+      throw new GradleException(errorMsg)
+    }
+  }
 }
 
 Task reportAllTestsResult = tasks.create (
@@ -1769,26 +1786,36 @@ Task reportAllTestsResult = tasks.create (
   type: Task
 ).with {
   doLast {
+    if (project.ext.testSessions.isEmpty()) {
+      return
+    }
+
+    println "\n\n[Test Summary]"
     int failedCount = 0
     int totalCount = 0
     project.ext.testSessions.each { session ->
       String resultStr
       ++totalCount
+      String logType = ""
       if (session.isPassed) {
         resultStr = "PASSED"
       } else {
         resultStr = "FAILED"
+        logType = "::error::"
         ++failedCount
       }
-      println sprintf("Test %s %s [%s/%s]",
+      println sprintf("%sTest %s %s [%s/%s]",
+          logType,
           session.name,
           resultStr,
           session.type,
           session.module)
     }
+    println "--------------------------------------"
+    println sprintf("::notice::%s out of %d test(s) passed", totalCount - failedCount, totalCount)
     if (failedCount > 0) {
       throw new GradleException(
-        sprintf("%d out of %d tests failed", failedCount, totalCount))
+        sprintf("%d out of %d test(s) failed", failedCount, totalCount))
     }
   }
 }
@@ -1805,6 +1832,9 @@ Task testPackageUploader = createPythonTask(
     doLast {
       EvaluateTestResult(task)
     }
+    doFirst {
+      ReportTestStarted(task)
+    }
     setTestProperties(task, TestTypeEnum.PYTHON, TestModuleEnum.TOOL)
   }
 
@@ -1821,7 +1851,10 @@ createPythonTask(
     doLast {
       EvaluateTestResult(task)
     }
-  }
+    doFirst {
+      ReportTestStarted(task)
+    }
+}
 
 createPythonTask(
   "testGenGuids",
@@ -1836,6 +1869,9 @@ createPythonTask(
     doLast {
       EvaluateTestResult(task)
     }
+    doFirst {
+      ReportTestStarted(task)
+    }
   }
 
 createPythonTask(
@@ -1851,6 +1887,9 @@ createPythonTask(
     doLast {
       EvaluateTestResult(task)
     }
+    doFirst {
+      ReportTestStarted(task)
+    }
   }
 
 task updateEmbeddedGradleWrapper(type: Zip) {

gha/build_setup/action.yml

@@ -65,7 +65,7 @@ runs:
       with:
         version: ${{ inputs.unity_version }}
         # iOS build support is always required to build EDM4U
-        platforms: "${{ inputs.platform }},iOS"
+        platforms: "${{ inputs.platform }},iOS,Android"
         username: ${{ inputs.unity_username }}
         password: ${{ inputs.unity_password }}
         serial_ids: ${{ inputs.unity_serial_id }}