Merge pull request #2852 from cdr/jsjoeio-2646-separate-testing

dev(testing): separate unit and e2e tests

Author: jsjoeio

.github/workflows/ci.yaml

@@ -27,7 +27,15 @@ jobs:
         with:
           args: ./ci/steps/lint.sh
 
-  test:
+  test-unit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Run unit tests
+        uses: ./ci/images/debian10
+        with:
+          args: ./ci/steps/test-unit.sh
+  test-e2e:
     needs: linux-amd64
     runs-on: ubuntu-latest
     env:
@@ -44,19 +52,19 @@ jobs:
         run: |
           cd release-packages && tar -xzf code-server*-linux-amd64.tar.gz
       - uses: microsoft/playwright-github-action@v1
-      - name: Install dependencies and run tests
+      - name: Install dependencies and run end-to-end tests
         run: |
           ./release-packages/code-server*-linux-amd64/bin/code-server --home $CODE_SERVER_ADDRESS/healthz &
           yarn --frozen-lockfile
-          yarn test
+          yarn test:e2e
       - name: Upload test artifacts
         if: always()
         uses: actions/upload-artifact@v2
         with:
           name: test-videos
-          path: ./test/videos
+          path: ./test/e2e/videos
       - name: Remove release packages and test artifacts
-        run: rm -rf ./release-packages ./test/videos
+        run: rm -rf ./release-packages ./test/e2e/videos
 
   release:
     runs-on: ubuntu-latest

.gitignore

@@ -16,5 +16,5 @@ node-*
 .home
 coverage
 **/.DS_Store
-test/videos
-test/screenshots
+test/e2e/videos
+test/e2e/screenshots

ci/README.md

@@ -52,7 +52,7 @@ Make sure you have `$GITHUB_TOKEN` set and [hub](https://github.com/github/hub)
 
 Currently, we run a command to manually generate the code coverage shield. Follow these steps:
 
-1. Run `yarn test` and make sure all the tests are passing
+1. Run `yarn test:unit` and make sure all the tests are passing
 2. Run `yarn badges`
 3. Go into the README and change the color from `red` to `green` in this line:
 
@@ -72,8 +72,10 @@ This directory contains scripts used for the development of code-server.
   - Runs formatters.
 - [./ci/dev/lint.sh](./dev/lint.sh) (`yarn lint`)
   - Runs linters.
-- [./ci/dev/test.sh](./dev/test.sh) (`yarn test`)
-  - Runs tests.
+- [./ci/dev/test-unit.sh](./dev/test-unit.sh) (`yarn test:unit`)
+  - Runs unit tests.
+- [./ci/dev/test-e2e.sh](./dev/test-e2e.sh) (`yarn test:e2e`)
+  - Runs end-to-end tests.
 - [./ci/dev/ci.sh](./dev/ci.sh) (`yarn ci`)
   - Runs `yarn fmt`, `yarn lint` and `yarn test`.
 - [./ci/dev/watch.ts](./dev/watch.ts) (`yarn watch`)
@@ -142,11 +144,13 @@ This directory contains the scripts used in CI.
 Helps avoid clobbering the CI configuration.
 
 - [./steps/fmt.sh](./steps/fmt.sh)
-  - Runs `yarn fmt` after ensuring VS Code is patched.
+  - Runs `yarn fmt`.
 - [./steps/lint.sh](./steps/lint.sh)
-  - Runs `yarn lint` after ensuring VS Code is patched.
-- [./steps/test.sh](./steps/test.sh)
-  - Runs `yarn test` after ensuring VS Code is patched.
+  - Runs `yarn lint`.
+- [./steps/test-unit.sh](./steps/test-unit.sh)
+  - Runs `yarn test:unit`.
+- [./steps/test-e2e.sh](./steps/test-e2e.sh)
+  - Runs `yarn test:e2e`.
 - [./steps/release.sh](./steps/release.sh)
   - Runs the release process.
   - Generates the npm package at `./release`.

ci/dev/ci.sh

@@ -6,7 +6,7 @@ main() {
 
   yarn fmt
   yarn lint
-  yarn test
+  yarn test:unit
 }
 
 main "$@"

ci/dev/test.sh renamed to ci/dev/test-e2e.sh

@@ -3,12 +3,9 @@ set -euo pipefail
 
 main() {
   cd "$(dirname "$0")/../.."
-  cd test/test-plugin
-  make -s out/index.js
   # We must keep jest in a sub-directory. See ../../test/package.json for more
   # information. We must also run it from the root otherwise coverage will not
   # include our source files.
-  cd "$OLDPWD"
   if [[ -z ${PASSWORD-} ]] || [[ -z ${CODE_SERVER_ADDRESS-} ]]; then
     echo "The end-to-end testing suites rely on your local environment"
     echo -e "\n"
@@ -21,7 +18,7 @@ main() {
     echo -e "\n"
     exit 1
   fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
 }
 
 main "$@"

ci/dev/test-unit.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+  cd "$(dirname "$0")/../.."
+  cd test/unit/test-plugin
+  make -s out/index.js
+  # We must keep jest in a sub-directory. See ../../test/package.json for more
+  # information. We must also run it from the root otherwise coverage will not
+  # include our source files.
+  cd "$OLDPWD"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+}
+
+main "$@"

ci/steps/test-e2e.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+  cd "$(dirname "$0")/../.."
+
+  "./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
+  yarn --frozen-lockfile
+  yarn test:e2e
+}
+
+main "$@"

ci/steps/test.sh renamed to ci/steps/test-unit.sh

@@ -6,7 +6,7 @@ main() {
 
   yarn --frozen-lockfile
 
-  yarn test
+  yarn test:unit
 }
 
 main "$@"

package.json

@@ -16,7 +16,9 @@
     "release:standalone": "./ci/build/build-standalone-release.sh",
     "release:github-draft": "./ci/build/release-github-draft.sh",
     "release:github-assets": "./ci/build/release-github-assets.sh",
+    "test:e2e": "./ci/dev/test-e2e.sh",
     "test:standalone-release": "./ci/build/test-standalone-release.sh",
+    "test:unit": "./ci/dev/test-unit.sh",
     "package": "./ci/build/build-packages.sh",
     "postinstall": "./ci/dev/postinstall.sh",
     "update:vscode": "./ci/dev/update-vscode.sh",
@@ -124,7 +126,8 @@
     "testPathIgnorePatterns": [
       "node_modules",
       "lib",
-      "out"
+      "out",
+      "test/e2e"
     ],
     "collectCoverage": true,
     "collectCoverageFrom": [
@@ -144,8 +147,6 @@
         "lines": 40
       }
     },
-    "testTimeout": 30000,
-    "globalSetup": "<rootDir>/test/globalSetup.ts",
     "modulePathIgnorePatterns": [
       "<rootDir>/lib/vscode",
       "<rootDir>/release-packages",
@@ -156,7 +157,7 @@
       "<rootDir>/release-images"
     ],
     "moduleNameMapper": {
-      "^.+\\.(css|less)$": "<rootDir>/test/cssStub.ts"
+      "^.+\\.(css|less)$": "<rootDir>/test/utils/cssStub.ts"
     }
   }
 }

test/e2e.test.ts renamed to test/e2e/e2e.test.ts

@@ -1,5 +1,5 @@
 import { chromium, Page, Browser } from "playwright"
-import { CODE_SERVER_ADDRESS } from "./constants"
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
 
 let browser: Browser
 let page: Page