Merge pull request #933 from coseeian/axe-core-playwright

Add Foundation for Automated Accessibility Testing

Author: ksen0

.gitignore

@@ -25,5 +25,8 @@ in/
 out/
 translator/
 
+playwright-report/
+test-results/
+
 # Don't commit custom dev builds
 public/p5.min.js

package-lock.json

@@ -17,7 +17,8 @@
     "build:search": "tsx ./src/scripts/builders/search.ts",
     "build:p5-version": "tsx ./src/scripts/p5-version.ts",
     "custom:dev": "tsx ./src/scripts/branchTest.ts",
-    "custom:cleanup": "tsx ./src/scripts/resetBranchTest.ts"
+    "custom:cleanup": "tsx ./src/scripts/resetBranchTest.ts",
+    "test:a11y": "playwright test test/a11y"
   },
   "dependencies": {
     "@astrojs/check": "^0.5.5",
@@ -51,7 +52,9 @@
     }
   },
   "devDependencies": {
+    "@axe-core/playwright": "^4.10.2",
     "@codemirror/lang-javascript": "^6.2.2",
+    "@playwright/test": "^1.54.1",
     "@preact/preset-vite": "^2.8.2",
     "@swc/html": "^1.10.9",
     "@testing-library/preact": "^3.2.3",

package.json

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from '@playwright/test';
+
+// --- Runtime modes ---
+// RUN_MODE=LOCAL   -> Start Astro dev server (local development)
+// RUN_MODE=BUILD   -> Build Astro site and serve from "dist" (CI / closest to production)
+// RUN_MODE=REMOTE  -> Do not start any local server, test directly against remote URL
+const RUN_MODE = process.env.RUN_MODE ?? (process.env.CI ? 'BUILD' : 'LOCAL');
+
+// Allow overriding test directory via environment variable (default: ./tests)
+const testDir = process.env.TEST_DIR ?? './test';
+
+// Base URL changes depending on the mode
+// LOCAL -> http://localhost:4321 (Astro dev server)
+// BUILD -> http://localhost:4173 (served "dist")
+// REMOTE -> PROD_BASE_URL (falls back to p5js.org)
+const baseURL =
+  RUN_MODE === 'LOCAL'
+    ? 'http://localhost:4321'
+    : RUN_MODE === 'BUILD'
+    ? 'http://localhost:4173'
+    : process.env.PROD_BASE_URL ?? 'https://p5js.org';
+
+
+export default defineConfig({
+  // Use dynamic testDir (default ./tests)
+  testDir,
+  outputDir: 'test-results',
+  // Global timeout for each test to improve stability
+  timeout: 30 * 1000,
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  // Retry failed tests in CI
+  retries: process.env.CI ? 2 : 0,
+  // Force single worker in CI to avoid port/resource conflicts
+  workers: process.env.CI ? 1 : undefined,
+  // Reporters: "list" for readable console logs + "html" for detailed report
+  reporter: [
+    ['list'],
+    ['html', { outputFolder: 'playwright-report', open: 'never' }],
+  ],
+  use: {
+    baseURL,
+    // Save trace only on first retry for debugging failed tests
+    trace: 'on-first-retry',
+    // Capture screenshot only on failure
+    screenshot: 'only-on-failure',
+    // Keep video only on failure in CI
+    video: process.env.CI ? 'retain-on-failure' : 'off',
+  },
+
+  // Test projects: three major engines + iPhone 15 viewport
+  projects: [
+    { name: 'Desktop Chrome', use: { ...devices['Desktop Chrome'] } },
+    { name: 'Desktop Firefox',  use: { ...devices['Desktop Firefox'] } },
+    { name: 'Desktop Safari',   use: { ...devices['Desktop Safari'] } },
+    { name: 'iPhone 15', use: { ...devices['iPhone 15'] } },
+    { name: 'Pixel 7', use: { ...devices['Pixel 7'] } },
+  ],
+
+  // Start appropriate webServer depending on the mode
+  webServer:
+    RUN_MODE === 'LOCAL'
+      ? {
+          // Start Astro dev server for local development
+          command: 'npm run dev',
+            port: 4321,
+          reuseExistingServer: !process.env.CI,
+          timeout: 180_000,
+        }
+      : RUN_MODE === 'BUILD'
+      ? {
+        command: 'npm run build && npm run preview -- --port 4173 --host',
+        port: 4173, // choose port OR url (not both)
+        reuseExistingServer: !process.env.CI,
+        timeout: 180_000,
+      }
+      : undefined, // REMOTE mode → no server started
+});
+

playwright.config.ts

@@ -0,0 +1,10 @@
+import AxeBuilder from "@axe-core/playwright";
+import { expect, test } from "@playwright/test";
+
+test.describe("a11y-home-page", () => {
+    test("should not have any automatically detectable accessibility issues", async ({ page }) => {
+        await page.goto("/");
+        const accessibilityScanResults = await new AxeBuilder({ page }).analyze();
+        expect(accessibilityScanResults.violations).toEqual([]);
+    });
+});

test/a11y/HomePage.test.ts

@@ -1,8 +1,9 @@
 /// <reference types="vitest" />
+import { defineConfig, configDefaults } from 'vitest/config';
 import { getViteConfig } from "astro/config";
 import preact from "@preact/preset-vite";
 
-export default getViteConfig({
+export default getViteConfig(defineConfig({
   // needed to ensure react 3rd party libraries work with preact
   plugins: [preact()],
   resolve: {
@@ -12,5 +13,9 @@ export default getViteConfig({
     /* for example, use global to avoid globals imports (describe, test, expect): */
     // globals: true,
     environment: "jsdom",
+    exclude: [
+      ...configDefaults.exclude,
+      'test/a11y/**',
+    ],
   },
-});
+}));