fix: improve stability of double pipeline tests (#5142)

* fix: improve stability of double pipeline tests and enhance snapshot regeneration logic

* chore: print snapshot changes

* chore: revert logs inside check.ts

* test: print changed snapshot variable

* fix: issue with snapshot changes

* fix: issue with cancel

* fix: cancel workflow

* fix: default.yml pipeline

* auto update snapshots (#5162)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* chore: add more output for failures

* fix: remove env. from if statement

* auto update snapshots (#5170)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: nmerget

.github/workflows/02-e2e-showcases.yml

@@ -77,8 +77,7 @@ jobs:
       # We test everything for patternhub and main branch
       - name: 👩‍🔬 Test showcase with Playwright 🎭
         shell: bash
-        # TODO: we need to find a more stable solution / https://github.com/db-ux-design-system/core-web/issues/4910
-        # if: github.event.pull_request == null
+        if: github.event.pull_request == null
         env:
           NEXT_PUBLIC_BASE_PATH: ${{ inputs.path }}
         run: |
@@ -92,9 +91,13 @@ jobs:
         run: |
           npm run test:aria-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --update-snapshots
           OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-aria.ts)
-          echo "aria-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
+          if [ "$OUTPUT" = "true" ]; then
+            echo "aria-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
+          fi
           OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-main-aria.ts)
-          echo "aria-snapshots-changed-to-main=$OUTPUT" >> $GITHUB_OUTPUT
+          if [ "$OUTPUT" = "true" ]; then
+            echo "aria-snapshots-changed-to-main=$OUTPUT" >> $GITHUB_OUTPUT
+          fi
 
       - name: 👩‍🔬 Test axe-core with Playwright 🎭
         shell: bash
@@ -109,16 +112,29 @@ jobs:
         run: |
           npm run test:visual-snapshots --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --update-snapshots
           OUTPUT=$(npx --no tsx scripts/github/snapshot-changes/check-visual.ts)
-          echo "visual-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
+          if [ "$OUTPUT" = "true" ]; then
+            echo "visual-snapshots-changed=$OUTPUT" >> $GITHUB_OUTPUT
+          fi
 
       - name: 👩‍🔬 Test a11y-checker with Playwright 🎭
         shell: bash
         if: |
           github.event.pull_request != null &&
-          steps.aria-snapshots.outputs.aria-snapshots-changed-to-main=='true'
+          steps.aria-snapshots.outputs.aria-snapshots-changed-to-main == 'true'
         run: |
           npm run test:a11y-checker --workspace=${{ inputs.showcase }} -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
+      - name: 🌳 Log Snapshot Changes
+        if: github.event.pull_request != null
+        env:
+          ARIA: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed}}
+          VISUAL: ${{ steps.visual-snapshots.outputs.visual-snapshots-changed}}
+          ARIA_MAIN: ${{ steps.aria-snapshots.outputs.aria-snapshots-changed-to-main }}
+        run: |
+          echo "ARIA: $ARIA"
+          echo "VISUAL: $VISUAL"
+          echo "ARIA_MAIN: $ARIA_MAIN"
+
       - name: 🔣 Print GitHub Report
         if: failure()
         shell: bash

.github/workflows/default.yml

@@ -272,7 +272,16 @@ jobs:
     with:
       version: ${{ needs.get-playwright-version.outputs.version }}
 
-  regenerate-snapshots:
+  should-regenerate-snapshots:
+    env:
+      ARIA_ANGULAR: ${{ needs.test-showcase-angular.outputs.aria-snapshots-changed }}
+      VISUAL_ANGULAR: ${{ needs.test-showcase-angular.outputs.visual-snapshots-changed }}
+      ARIA_REACT: ${{ needs.test-showcase-react.outputs.aria-snapshots-changed }}
+      VISUAL_REACT: ${{ needs.test-showcase-react.outputs.visual-snapshots-changed }}
+      ARIA_VUE: ${{ needs.test-showcase-vue.outputs.aria-snapshots-changed }}
+      VISUAL_VUE: ${{ needs.test-showcase-vue.outputs.visual-snapshots-changed }}
+      ARIA_STENCIL: ${{ needs.test-showcase-stencil.outputs.aria-snapshots-changed }}
+      VISUAL_STENCIL: ${{ needs.test-showcase-stencil.outputs.visual-snapshots-changed }}
     if: |
       !cancelled() && (
         needs.test-showcase-angular.result == 'failure' ||
@@ -283,19 +292,41 @@ jobs:
         needs.test-showcase-react.outputs.aria-snapshots-changed == 'true' ||
         needs.test-showcase-react.outputs.visual-snapshots-changed == 'true' ||
         needs.test-showcase-vue.outputs.aria-snapshots-changed == 'true' ||
-        needs.test-showcase-vue.outputs.visual-snapshots-changed == 'true'
+        needs.test-showcase-vue.outputs.visual-snapshots-changed == 'true' ||
+        needs.test-showcase-stencil.outputs.aria-snapshots-changed == 'true' ||
+        needs.test-showcase-stencil.outputs.visual-snapshots-changed == 'true'
       )
-    uses: ./.github/workflows/02-e2e-regenerate.yml
-    with:
-      version: ${{ needs.get-playwright-version.outputs.version }}
-      type: showcases
     needs:
       [
         test-showcase-angular,
         test-showcase-react,
         test-showcase-vue,
-        get-playwright-version
+        test-showcase-stencil
       ]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: "Fail if actor is a bot and regeneration is triggered"
+        run: |
+          if [[ "${{ github.actor }}" == *"[bot]"* ]]; then
+              echo "The snapshots between frameworks are different." >> $GITHUB_STEP_SUMMARY
+              echo "| Framework | Aria Snapshots Changed | Visual Snapshots Changed |" >> $GITHUB_STEP_SUMMARY
+              echo "|-----------|------------------------|--------------------------|" >> $GITHUB_STEP_SUMMARY
+              echo "| Angular   | "$ARIA_ANGULAR"  | "$VISUAL_ANGULAR" |" >> $GITHUB_STEP_SUMMARY
+              echo "| React     | "$ARIA_REACT"    | "$VISUAL_REACT"   |" >> $GITHUB_STEP_SUMMARY
+              echo "| Vue       | "$ARIA_VUE"      | "$VISUAL_VUE"     |" >> $GITHUB_STEP_SUMMARY
+              echo "| Stencil   | "$ARIA_STENCIL"  | "$VISUAL_STENCIL" |" >> $GITHUB_STEP_SUMMARY
+              exit 1
+          fi
+        shell: bash
+
+  regenerate-snapshots:
+    if: |
+      !cancelled() && needs.should-regenerate-snapshots.result == 'success'
+    uses: ./.github/workflows/02-e2e-regenerate.yml
+    with:
+      version: ${{ needs.get-playwright-version.outputs.version }}
+      type: showcases
+    needs: [should-regenerate-snapshots, get-playwright-version]
 
   commit-regenerated-snapshots:
     uses: ./.github/workflows/02-e2e-regenerated-snapshots-commit.yml