fix(testing): Checkout proper branch for Wokwi tests and small QoL improvements (#10435)

* fix(tests): Add missing newlines to output

* fix(tests): Improve fibonacci test

* fix(tests): Remove redundant targets from json

* fix(wokwi): Checkout proper base branch for wokwi tests

* feat(logging): Add logging to some tests to improve debugging

* fix(ci): Make CI more permissive and improve messages

* fix(tests): Bump pytest-embedded version to support P4

Author: lucasssvaz

.github/scripts/tests_matrix.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+build_types="'validation'"
+hw_types="'validation'"
+wokwi_types="'validation'"
+qemu_types="'validation'"
+
+if [[ $IS_PR != 'true' ]] || [[ $PERFORMANCE_ENABLED == 'true' ]]; then
+  build_types+=",'performance'"
+  hw_types+=",'performance'"
+  #wokwi_types+=",'performance'"
+  #qemu_types+=",'performance'"
+fi
+
+targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2'"
+
+mkdir -p info
+
+echo "[$wokwi_types]" > info/wokwi_types.txt
+echo "[$targets]" > info/targets.txt
+
+echo "build-types=[$build_types]" >> $GITHUB_OUTPUT
+echo "hw-types=[$hw_types]" >> $GITHUB_OUTPUT
+echo "wokwi-types=[$wokwi_types]" >> $GITHUB_OUTPUT
+echo "qemu-types=[$qemu_types]" >> $GITHUB_OUTPUT
+echo "targets=[$targets]" >> $GITHUB_OUTPUT

.github/workflows/dangerjs.yml

@@ -19,4 +19,6 @@ jobs:
     - name: DangerJS pull request linter
       uses: espressif/shared-github-dangerjs@v1
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        rule-max-commits: 'false'
+        commit-messages-min-summary-length: '10'

.github/workflows/tests.yml

@@ -57,29 +57,25 @@ jobs:
       hw-types: ${{ steps.set-matrix.outputs.hw-types }}
       wokwi-types: ${{ steps.set-matrix.outputs.wokwi-types }}
       qemu-types: ${{ steps.set-matrix.outputs.qemu-types }}
+      targets: ${{ steps.set-matrix.outputs.targets }}
+    env:
+      IS_PR: ${{ github.event.pull_request.number != null }}
+      PERFORMANCE_ENABLED: ${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts/tests_matrix.sh
+
       - name: Set matrix
         id: set-matrix
-        run: |
-          build_types='["validation"'
-          hw_types='["validation"'
-          wokwi_types='["validation"'
-          qemu_types='["validation"'
-
-          is_pr=${{ github.event.pull_request.number != null }}
-          is_performance_enabled=${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }}
-
-          if [[ $is_pr != 'true' ]] || [[ $is_performance_enabled == 'true' ]]; then
-            build_types+=',"performance"'
-            hw_types+=',"performance"'
-            #wokwi_types+=',"performance"'
-            #qemu_types+=',"performance"'
-          fi
+        run: bash .github/scripts/tests_matrix.sh
 
-          echo "build-types=$build_types]" >> $GITHUB_OUTPUT
-          echo "hw-types=$hw_types]" >> $GITHUB_OUTPUT
-          echo "wokwi-types=$wokwi_types]" >> $GITHUB_OUTPUT
-          echo "qemu-types=$qemu_types]" >> $GITHUB_OUTPUT
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: matrix_info
+          path: info/*
 
   call-build-tests:
     name: Build
@@ -88,7 +84,7 @@ jobs:
     strategy:
       matrix:
         type: ${{ fromJson(needs.gen-matrix.outputs.build-types) }}
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }}
     with:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}
@@ -105,7 +101,7 @@ jobs:
       fail-fast: false
       matrix:
         type: ${{ fromJson(needs.gen-matrix.outputs.hw-types) }}
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }}
     with:
       type: ${{ matrix.type }}
       chip: ${{ matrix.chip }}

.github/workflows/tests_results.yml

@@ -79,6 +79,7 @@ jobs:
           event_name: ${{ env.original_event }}
           files: ./artifacts/**/*.xml
           action_fail: true
+          compare_to_earlier_commit: false
 
       - name: Fail if tests failed
         if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }}

.github/workflows/tests_wokwi.yml

@@ -22,6 +22,9 @@ jobs:
     outputs:
       pr_num: ${{ steps.set-ref.outputs.pr_num }}
       ref: ${{ steps.set-ref.outputs.ref }}
+      base: ${{ steps.set-ref.outputs.base }}
+      targets: ${{ steps.set-ref.outputs.targets }}
+      types: ${{ steps.set-ref.outputs.types }}
     steps:
       - name: Report pending
         uses: actions/github-script@v7
@@ -51,10 +54,18 @@ jobs:
           name: event_file
           path: artifacts/event_file
 
+      - name: Download and extract matrix info
+        uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          name: matrix_info
+          path: artifacts/matrix_info
+
       - name: Try to read PR number
         id: set-ref
         run: |
-          pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json)
+          pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json | tr -cd "[:digit:]")
           if [ -z "$pr_num" ] || [ "$pr_num" == "null" ]; then
             pr_num=""
           fi
@@ -64,11 +75,22 @@ jobs:
             ref=${{ github.ref }}
           fi
 
-          action=$(jq -r '.action' artifacts/event_file/event.json)
+          action=$(jq -r '.action' artifacts/event_file/event.json | tr -cd "[:alpha:]_")
           if [ "$action" == "null" ]; then
             action=""
           fi
 
+          base=$(jq -r '.pull_request.base.ref' artifacts/event_file/event.json | tr -cd "[:alnum:]/_.-")
+          if [ -z "$base" ] || [ "$base" == "null" ]; then
+            base=${{ github.ref }}
+          fi
+
+          types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'")
+          targets=$(cat artifacts/matrix_info/targets.txt | tr -cd "[:alnum:],[]'")
+
+          echo "base = $base"
+          echo "targets = $targets"
+          echo "types = $types"
           echo "pr_num = $pr_num"
 
           printf "$ref" >> artifacts/ref.txt
@@ -98,6 +120,9 @@ jobs:
           cat artifacts/conclusion.txt
 
           echo "pr_num=$pr_num" >> $GITHUB_OUTPUT
+          echo "base=$base" >> $GITHUB_OUTPUT
+          echo "targets=$targets" >> $GITHUB_OUTPUT
+          echo "types=$types" >> $GITHUB_OUTPUT
           echo "ref=$ref" >> $GITHUB_OUTPUT
 
       - name: Download and extract parent hardware results
@@ -164,8 +189,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        type: ['validation']
-        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
+        type: ${{ fromJson(needs.get-artifacts.outputs.types) }}
+        chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }}
     steps:
       - name: Report pending
         uses: actions/github-script@v7
@@ -211,9 +236,12 @@ jobs:
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
       # Note that changes to the workflows and tests will only be picked up after the PR is merged
+      # DO NOT CHECKOUT THE USER'S REPOSITORY IN THIS WORKFLOW. IT HAS HIGH SECURITY RISKS.
       - name: Checkout repository
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.get-artifacts.outputs.base || github.ref }}
 
       - uses: actions/setup-python@v5
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}

tests/performance/coremark/coremark.ino

@@ -40,7 +40,7 @@ void setup() {
   Serial.printf("Cores: %d\n", CONFIG_SOC_CPU_CORES_NUM);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     coremark_main();
     Serial.flush();
   }

tests/performance/fibonacci/fibonacci.ino

@@ -31,7 +31,7 @@ void setup() {
   Serial.printf("N: %d\n", FIB_N);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     unsigned long start = millis();
     fibonacci = fib(FIB_N);
     unsigned long elapsed = millis() - start;

tests/performance/fibonacci/test_fibonacci.py

@@ -2,24 +2,21 @@
 import logging
 import os
 
+fib_results = {}
+
+def fib(n):
+    if n < 2:
+        return n
+    elif str(n) in fib_results:
+        return fib_results[str(n)]
+    else:
+        fib_results[str(n)] = fib(n - 1) + fib(n - 2)
+        return fib_results[str(n)]
+
 
 def test_fibonacci(dut, request):
     LOGGER = logging.getLogger(__name__)
 
-    # Fibonacci results starting from fib(35) to fib(45)
-    fib_results = [
-        9227465,
-        14930352,
-        24157817,
-        39088169,
-        63245986,
-        102334155,
-        165580141,
-        267914296,
-        433494437,
-        701408733,
-    ]
-
     # Match "Runs: %d"
     res = dut.expect(r"Runs: (\d+)", timeout=60)
     runs = int(res.group(0).decode("utf-8").split(" ")[1])
@@ -30,7 +27,11 @@ def test_fibonacci(dut, request):
     res = dut.expect(r"N: (\d+)", timeout=300)
     fib_n = int(res.group(0).decode("utf-8").split(" ")[1])
     LOGGER.info("Calculating Fibonacci({})".format(fib_n))
-    assert fib_n > 30 and fib_n < 50, "Invalid Fibonacci number"
+    assert fib_n > 0, "Invalid Fibonacci number"
+
+    # Calculate Fibonacci results
+    expected_result = fib(fib_n)
+    LOGGER.info("Expected Fibonacci result: {}".format(expected_result))
 
     list_time = []
 
@@ -48,7 +49,7 @@ def test_fibonacci(dut, request):
         assert fib_result > 0, "Invalid Fibonacci result"
 
         # Check if the result is correct
-        assert fib_result == fib_results[fib_n - 35]
+        assert fib_result == expected_result
 
         # Match "Time: %lu.%03lu s"
         res = dut.expect(r"Time: (\d+)\.(\d+) s", timeout=300)

tests/performance/psramspeed/ci.json

@@ -3,9 +3,7 @@
     "qemu": false,
     "wokwi": false
   },
-  "targets": {
-    "esp32c3": false,
-    "esp32c6": false,
-    "esp32h2": false
-  }
+  "requires": [
+    "CONFIG_SPIRAM=y"
+  ]
 }

tests/performance/psramspeed/psramspeed.ino

@@ -252,7 +252,7 @@ void setup() {
   Serial.printf("Max test size: %d\n", MAX_TEST_SIZE);
   Serial.flush();
   for (int i = 0; i < N_RUNS; i++) {
-    Serial.printf("Run %d", i);
+    Serial.printf("Run %d\n", i);
     memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES);
     Serial.flush();
     memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES);