Add visual test structure (#3)

* Add visual test structure

* Fix configuration

* Add webkit snapshots

* Upload artifact only on failure

* Fix MANIFEST

* Skip autofocus for now

Author: fcollonval

.github/workflows/ci.yml

@@ -34,6 +34,56 @@ jobs:
         run: yarn run test
         working-directory: packages/components
 
+  visual-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v2
+
+      - name: Setup Node 💾
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Setup yarn cache
+        uses: actions/cache@v2
+        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            yarn-
+
+      - name: Install Dependencies 📥
+        run: yarn install
+
+      - name: Install Playwright Browsers
+        run: yarn run playwright install --with-deps
+        working-directory: packages/components
+
+      - name: Run Visual Tests 🧪
+        run: yarn run test:visual
+        working-directory: packages/components
+
+      - name: Update Snapshots
+        if: ${{ failure() }}
+        run: |
+          mv test-results test-assets
+          yarn run test:visual -u
+        working-directory: packages/components
+
+      - uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: jupyter-ui-test
+          path: |
+            packages/components/test-assets/
+            packages/components/src/**/*-snapshots/*
+
   lint:
     runs-on: ubuntu-latest
     timeout-minutes: 10

.gitignore

@@ -128,3 +128,6 @@ yarn-error.log*
 
 # Cache
 temp
+
+# Visual testing
+packages/components/test-results

MANIFEST.in

@@ -14,6 +14,8 @@ graft jupyter_ui_demo/labextension
 # Javascript files
 include packages/components/api-extractor.json
 include packages/components/package.json
+include packages/components/jest.config.js
+include packages/components/playwright.config.ts
 include packages/components/rollup.config.js
 include packages/components/tsconfig.json
 graft packages/components/.storybook

README.md

@@ -4,7 +4,7 @@
 
 ![Extension status](https://img.shields.io/badge/status-draft-critical 'Not yet working')
 [![NPM Version](https://img.shields.io/npm/v/@jupyter-notebook/web-components?color=blue)](https://www.npmjs.com/package/@jupyter-notebook/web-components)
-![Toolkit CI Status](https://github.com/jupyterlab-contrib/jupyter-ui-toolkit/actions/workflows/ci.yml/badge.svg)
+[![Toolkit CI Status](https://github.com/jupyterlab-contrib/jupyter-ui-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/jupyterlab-contrib/jupyter-ui-toolkit/actions/workflows/ci.yml)
 ![Deploy Docs Status](https://github.com/jupyterlab-contrib/jupyter-ui-toolkit/actions/workflows/docs-cd.yml/badge.svg)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jupyterlab-contrib/jupyter-ui-toolkit/main)
 

packages/components/.eslintignore

@@ -17,3 +17,4 @@ src/utilities/storybook/
 src/**/*.spec.ts
 src/**/*.stories.ts
 src/**/fixtures/
+playwright.config.ts

packages/components/jest.config.js

@@ -0,0 +1,5 @@
+const config = {
+  testMatch: ['**/?(*.)+(spec).ts']
+};
+
+module.exports = config;

packages/components/package.json

@@ -26,7 +26,8 @@
     "eslint:check": "eslint . --ext .ts",
     "eslint": "eslint . --ext .ts --fix",
     "prepublishOnly": "yarn run build",
-    "test": "jest --verbose --coverage"
+    "test": "jest --verbose --coverage",
+    "test:visual": "playwright test"
   },
   "dependencies": {
     "@microsoft/fast-element": "^1.6.0",
@@ -39,6 +40,7 @@
     "@babel/preset-typescript": "^7.13.0",
     "@microsoft/api-extractor": "^7.18.9",
     "@microsoft/eslint-config-fast-dna": "^1.2.0",
+    "@playwright/test": "^1.17.0",
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-node-resolve": "^11.2.0",
     "@rollup/plugin-typescript": "^8.2.0",

packages/components/playwright.config.ts

@@ -0,0 +1,34 @@
+import { PlaywrightTestConfig, devices } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  testMatch: '**/*.test.ts',
+  webServer: {
+    command: 'yarn run start',
+    port: 6006,
+    timeout: 120 * 1000,
+    // It is safe to reuse the server for stories testing
+    reuseExistingServer: true
+  },
+  use: {
+    baseURL: process.env.TARGET_URL ?? 'http://localhost:6006',
+    trace: 'on-first-retry'
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] }
+    },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] }
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] }
+    }
+  ]
+};
+
+export default config;

packages/components/src/badge/badge.test.ts

@@ -0,0 +1,11 @@
+import { test, expect } from '@playwright/test';
+
+test.describe('Badge', () => {
+  test('Default', async ({ page }) => {
+    await page.goto('/iframe.html?id=library-badge--default');
+
+    expect(
+      await page.locator('#root :nth-child(1)').first().screenshot()
+    ).toMatchSnapshot('badge-default.png');
+  });
+});