Reporter + weights

Author: mrginglymus

libraries/__shared__/tests/advanced-tests.test.js

@@ -16,10 +16,11 @@
  */
 
 import {expect, test} from '@playwright/test';
-import {getProp, each} from "./util";
+import {getProp, each, weight} from "./util";
+
 
 each(() => {
-  test.describe('advanced support', () => {
+  test.describe('advanced support',  weight(2), () => {
     test.describe('attributes and properties', () => {
       test('will pass array data as a property', async ({page}) => {
         const ce = page.locator('#ce-with-properties');
@@ -37,9 +38,9 @@ each(() => {
       })
     });
 
-    test.describe('events', () => {
+    test.describe('events', weight(1), () => {
 
-      test("can declaratively listen to a lowercase DOM event dispatched by a Custom Element", async ({page}) => {
+      test("can declaratively listen to a lowercase DOM event dispatched by a Custom Element", weight(2), async ({page}) => {
         const ce = page.locator('#ce-with-declarative-event');
         await expect(page.getByText(/lowercase:/)).toHaveText(/false/);
         await ce.click();

libraries/__shared__/tests/basic-tests.test.js

@@ -16,11 +16,11 @@
  */
 
 import {expect, test} from '@playwright/test';
-import {each, getPropOrAttr} from './util';
+import {each, getPropOrAttr, weight} from './util';
 
 
 each(() => {
-  test.describe("basic support", () => {
+  test.describe("basic support", weight(3), () => {
     test.describe("no children", () => {
       test("can display a Custom Element with no children", async ({page}) => {
         await expect(page.locator('#ce-without-children')).toBeAttached();

libraries/__shared__/tests/playwright.config.js

@@ -9,12 +9,17 @@ export default defineConfig({
       name: 'chromium',
       use: { ...devices['Desktop Chrome'] },
     },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
   ],
   webServer: {
     command: 'npm run serve',
     url: 'http://localhost:8080',
     reuseExistingServer: !process.env.CI,
     stdout: 'ignore',
     stderr: 'pipe'
-  }
+  },
+  reporter: './reporter.js'
 })

libraries/__shared__/tests/reporter.js

@@ -0,0 +1,47 @@
+import fs from 'node:fs';
+
+
+export default class CEEReporter {
+
+  results = {};
+
+  onTestEnd(test, result) {
+    const [_, _browser, _file, framework, suite] = test.titlePath();
+    ((this.results[framework] ||= {})[suite] ||= {})[result.status] ||= 0
+    this.results[framework][suite][result.status] += 1;
+    const weight = test.annotations.filter(({type}) => type === 'weight').toReversed()[0]?.description ?? 3;
+    (this.results[framework].weight ||= {})[result.status] ||= 0;
+    this.results[framework].weight[result.status] += weight
+  }
+
+  onEnd(result) {
+    for (const [framework, results] of Object.entries(this.results)) {
+      const basicPassed = results['basic support'].passed ?? 0;
+      const basicFailed = results['basic support'].failed ?? 0;
+      const advancedPassed = results['advanced support'].passed ?? 0;
+      const advancedFailed = results['advanced support'].failed ?? 0;
+
+      const weightedPassed = results.weight.passed ?? 0;
+      const weightedFailed = results.weight.failed ?? 0;
+      const totalWeight = weightedPassed + weightedFailed;
+
+      fs.writeFileSync(`../../${framework}/results/results.json`, JSON.stringify({
+        summary: {
+          success: basicPassed + advancedPassed,
+          failed: basicFailed + advancedFailed,
+          score: totalWeight === 0 ? 0 : 100 * weightedPassed / totalWeight,
+          basicSupport: {
+            total: basicPassed + basicFailed,
+            failed: basicFailed,
+            passed: basicPassed
+          },
+          advancedSupport: {
+            total: advancedPassed + advancedFailed,
+            failed: advancedFailed,
+            passed: advancedPassed
+          }
+        }
+      }, null, 2))
+    }
+  }
+}

libraries/__shared__/tests/util.js

@@ -9,6 +9,10 @@ export const getPropOrAttr = async (ce, name) => {
   return await getProp(ce, name) ?? ce.getAttribute(name);
 }
 
+export const weight = (weight) => ({
+  annotation: {type: 'weight', description: weight}
+})
+
 export const each = (cb) => {
   fs.readdirSync('./harness', {withFileTypes: true}).filter(d => d.isDirectory()).forEach(({name}) => {
     test.describe(name, () => {