Remove docker and user tests, related infra (#53118)

Author: jakebailey

.github/workflows/accept-baselines-fix-lints.yaml

@@ -16,7 +16,7 @@ jobs:
         git config user.email "typescriptbot@microsoft.com"
         git config user.name "TypeScript Bot"
         npm install
-        git rm -r --quiet tests/baselines/reference :^tests/baselines/reference/docker :^tests/baselines/reference/user
+        git rm -r --quiet tests/baselines/reference
         npx hereby runtests-parallel --ci --fix || true
         npx hereby baseline-accept
         git add ./src

.gitignore

@@ -34,7 +34,6 @@ tests/webTestServer.js.map
 tests/webhost/*.d.ts
 tests/webhost/webtsc.js
 tests/cases/**/*.js
-!tests/cases/docker/*.js/
 tests/cases/**/*.js.map
 *.config
 scripts/eslint/built/
@@ -59,29 +58,9 @@ internal/
 yarn.lock
 yarn-error.log
 .parallelperf.*
-tests/cases/user/*/package-lock.json
-tests/cases/user/*/node_modules/
-tests/cases/user/*/**/*.js
-tests/cases/user/*/**/*.js.map
-tests/cases/user/*/**/*.d.ts
-!tests/cases/user/zone.js/
-!tests/cases/user/bignumber.js/
-!tests/cases/user/discord.js/
 tests/baselines/reference/dt
 .failed-tests
 TEST-results.xml
 package-lock.json
-tests/cases/user/npm/npm
-tests/cases/user/TypeScript-React-Starter/TypeScript-React-Starter
-tests/cases/user/TypeScript-Node-Starter/TypeScript-Node-Starter
-tests/cases/user/TypeScript-React-Native-Starter/TypeScript-React-Native-Starter
-tests/cases/user/TypeScript-Vue-Starter/TypeScript-Vue-Starter
-tests/cases/user/TypeScript-WeChat-Starter/TypeScript-WeChat-Starter
-tests/cases/user/create-react-app/create-react-app
-tests/cases/user/fp-ts/fp-ts
-tests/cases/user/webpack/webpack
-tests/cases/user/puppeteer/puppeteer
-tests/cases/user/axios-src/axios-src
-tests/cases/user/prettier/prettier
 .eslintcache
-*v8.log
+*v8.log

CONTRIBUTING.md

@@ -81,10 +81,7 @@ hereby tests             # Build the test infrastructure using the built compile
 hereby runtests          # Run tests using the built compiler and test infrastructure.
                          # You can override the specific suite runner used or specify a test for this command.
                          # Use --tests=<testPath> for a specific test and/or --runner=<runnerName> for a specific suite.
-                         # Valid runners include conformance, compiler, fourslash, project, user, and docker
-                         # The user and docker runners are extended test suite runners - the user runner
-                         # works on disk in the tests/cases/user directory, while the docker runner works in containers.
-                         # You'll need to have the docker executable in your system path for the docker runner to work.
+                         # Valid runners include conformance, compiler, fourslash, and project
 hereby runtests-parallel # Like runtests, but split across multiple threads. Uses a number of threads equal to the system
                          # core count by default. Use --workers=<number> to adjust this.
 hereby baseline-accept   # This replaces the baseline test results with the results obtained from hereby runtests.

src/harness/runnerbase.ts

@@ -5,7 +5,7 @@ import {
 } from "./_namespaces/Harness";
 import * as ts from "./_namespaces/ts";
 
-export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "user" | "dt" | "docker";
+export type TestRunnerKind = CompilerTestKind | FourslashTestKind | "project" | "rwc" | "test262" | "dt";
 export type CompilerTestKind = "conformance" | "compiler";
 export type FourslashTestKind = "fourslash" | "fourslash-shims" | "fourslash-shims-pp" | "fourslash-server";
 

src/testRunner/externalCompileRunner.ts

@@ -113,180 +113,6 @@ abstract class ExternalCompileRunnerBase extends RunnerBase {
     }
 }
 
-export class UserCodeRunner extends ExternalCompileRunnerBase {
-    readonly testDir = "tests/cases/user/";
-    kind(): TestRunnerKind {
-        return "user";
-    }
-    report(result: ExecResult) {
-        // eslint-disable-next-line no-null/no-null
-        return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
-Standard output:
-${sortErrors(stripAbsoluteImportPaths(result.stdout.toString().replace(/\r\n/g, "\n")))}
-
-
-Standard error:
-${stripAbsoluteImportPaths(result.stderr.toString().replace(/\r\n/g, "\n"))}`;
-    }
-}
-
-export class DockerfileRunner extends ExternalCompileRunnerBase {
-    readonly testDir = "tests/cases/docker/";
-    kind(): TestRunnerKind {
-        return "docker";
-    }
-    override initializeTests(): void {
-        // Read in and evaluate the test list
-        const testList = this.tests && this.tests.length ? this.tests : this.getTestFiles();
-
-        // eslint-disable-next-line @typescript-eslint/no-this-alias
-        const cls = this;
-        describe(`${this.kind()} code samples`, function (this: Mocha.Suite) {
-            this.timeout(cls.timeout); // 20 minutes
-            before(() => {
-                // cached because workspace is hashed to determine cacheability
-                cls.exec("docker", ["build", ".", "-t", "typescript/typescript", "-f", cls.testDir + "Dockerfile"], {
-                    cwd: IO.getWorkspaceRoot(),
-                    env: { ...process.env, DOCKER_BUILDKIT: "1" }, // We need buildkit to allow Dockerfile.dockerignore to work.
-                });
-            });
-            for (const test of testList) {
-                const directory = typeof test === "string" ? test : test.file;
-                const cwd = path.join(IO.getWorkspaceRoot(), cls.testDir, directory);
-                it(`should build ${directory} successfully`, () => {
-                    const imageName = `tstest/${directory}`;
-                    cls.exec("docker", ["build", "--no-cache", ".", "-t", imageName], { cwd }); // --no-cache so the latest version of the repos referenced is always fetched
-                    const cp: typeof import("child_process") = require("child_process");
-                    Baseline.runBaseline(`${cls.kind()}/${directory}.log`, cls.report(cp.spawnSync(`docker`, ["run", imageName], { cwd, timeout: cls.timeout, shell: true })));
-                });
-            }
-        });
-    }
-
-    private timeout = 1_200_000; // 20 minutes;
-    private exec(command: string, args: string[], options: { cwd: string; env?: NodeJS.ProcessEnv }): void {
-        const cp: typeof import("child_process") = require("child_process");
-        const stdio = isWorker ? "pipe" : "inherit";
-        const res = cp.spawnSync(isWorker ? `${command} 2>&1` : command, args, { timeout: this.timeout, shell: true, stdio, ...options });
-        if (res.status !== 0) {
-            throw new Error(`${command} ${args.join(" ")} for ${options.cwd} failed: ${res.stdout && res.stdout.toString()}`);
-        }
-    }
-    report(result: ExecResult) {
-        // eslint-disable-next-line no-null/no-null
-        return result.status === 0 && !result.stdout.length && !result.stderr.length ? null : `Exit Code: ${result.status}
-Standard output:
-${sanitizeDockerfileOutput(result.stdout.toString())}
-
-
-Standard error:
-${sanitizeDockerfileOutput(result.stderr.toString())}`;
-    }
-}
-
-function sanitizeDockerfileOutput(result: string): string {
-    return [
-        normalizeNewlines,
-        stripANSIEscapes,
-        stripRushStageNumbers,
-        stripWebpackHash,
-        sanitizeVersionSpecifiers,
-        sanitizeTimestamps,
-        sanitizeSizes,
-        sanitizeUnimportantGulpOutput,
-        stripAbsoluteImportPaths,
-    ].reduce((result, f) => f(result), result);
-}
-
-function normalizeNewlines(result: string): string {
-    return result.replace(/\r\n/g, "\n");
-}
-
-function stripANSIEscapes(result: string): string {
-    return result.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
-}
-
-function stripRushStageNumbers(result: string): string {
-    return result.replace(/\d+ of \d+:/g, "XX of XX:");
-}
-
-function stripWebpackHash(result: string): string {
-    return result.replace(/Hash: \w+/g, "Hash: [redacted]");
-}
-
-function sanitizeSizes(result: string): string {
-    return result.replace(/\d+(\.\d+)? ((Ki|M)B|bytes)/g, "X KiB");
-}
-
-/**
- * Gulp's output order within a `parallel` block is nondeterministic (and there's no way to configure it to execute in series),
- * so we purge as much of the gulp output as we can
- */
-function sanitizeUnimportantGulpOutput(result: string): string {
-    return result.replace(/^.*(\] (Starting)|(Finished)).*$/gm, "") // "gulp" task start/end messages (nondeterministic order)
-        .replace(/^.*(\] . (finished)|(started)).*$/gm, "") // "just" task start/end messages (nondeterministic order)
-        .replace(/^.*\] Respawned to PID: \d+.*$/gm, "") // PID of child is OS and system-load dependent (likely stableish in a container but still dangerous)
-        .replace(/\n+/g, "\n")
-        .replace(/\/tmp\/yarn--.*?\/node/g, "");
-}
-
-function sanitizeTimestamps(result: string): string {
-    return result.replace(/\[\d?\d:\d\d:\d\d (A|P)M\]/g, "[XX:XX:XX XM]")
-        .replace(/\[\d?\d:\d\d:\d\d\]/g, "[XX:XX:XX]")
-        .replace(/\/\d+-\d+-[\d_TZ]+-debug.log/g, "\/XXXX-XX-XXXXXXXXX-debug.log")
-        .replace(/\d+\/\d+\/\d+ \d+:\d+:\d+ (AM|PM)/g, "XX/XX/XX XX:XX:XX XM")
-        .replace(/\d+(\.\d+)? sec(onds?)?/g, "? seconds")
-        .replace(/\d+(\.\d+)? min(utes?)?/g, "")
-        .replace(/\d+(\.\d+)? ?m?s/g, "?s")
-        .replace(/ \(\?s\)/g, "");
-}
-
-function sanitizeVersionSpecifiers(result: string): string {
-    return result
-        .replace(/\d+.\d+.\d+-insiders.\d\d\d\d\d\d\d\d/g, "X.X.X-insiders.xxxxxxxx")
-        .replace(/Rush Multi-Project Build Tool (\d+)\.\d+\.\d+/g, "Rush Multi-Project Build Tool $1.X.X")
-        .replace(/([@v\()])\d+\.\d+\.\d+/g, "$1X.X.X")
-        .replace(/webpack (\d+)\.\d+\.\d+/g, "webpack $1.X.X")
-        .replace(/Webpack version: (\d+)\.\d+\.\d+/g, "Webpack version: $1.X.X");
-}
-
-/**
- * Import types and some other error messages use absolute paths in errors as they have no context to be written relative to;
- * This is problematic for error baselines, so we grep for them and strip them out.
- */
-function stripAbsoluteImportPaths(result: string) {
-    const workspaceRegexp = new RegExp(IO.getWorkspaceRoot().replace(/\\/g, "\\\\"), "g");
-    return result
-        .replace(/import\(".*?\/tests\/cases\/user\//g, `import("/`)
-        .replace(/Module '".*?\/tests\/cases\/user\//g, `Module '"/`)
-        .replace(workspaceRegexp, "../../..");
-}
-
-function sortErrors(result: string) {
-    return ts.flatten(splitBy(result.split("\n"), s => /^\S+/.test(s)).sort(compareErrorStrings)).join("\n");
-}
-
-const errorRegexp = /^(.+\.[tj]sx?)\((\d+),(\d+)\)(: error TS.*)/;
-function compareErrorStrings(a: string[], b: string[]) {
-    ts.Debug.assertGreaterThanOrEqual(a.length, 1);
-    ts.Debug.assertGreaterThanOrEqual(b.length, 1);
-    const matchA = a[0].match(errorRegexp);
-    if (!matchA) {
-        return -1;
-    }
-    const matchB = b[0].match(errorRegexp);
-    if (!matchB) {
-        return 1;
-    }
-    const [, errorFileA, lineNumberStringA, columnNumberStringA, remainderA] = matchA;
-    const [, errorFileB, lineNumberStringB, columnNumberStringB, remainderB] = matchB;
-    return ts.comparePathsCaseSensitive(errorFileA, errorFileB) ||
-        ts.compareValues(parseInt(lineNumberStringA), parseInt(lineNumberStringB)) ||
-        ts.compareValues(parseInt(columnNumberStringA), parseInt(columnNumberStringB)) ||
-        ts.compareStringsCaseSensitive(remainderA, remainderB) ||
-        ts.compareStringsCaseSensitive(a.slice(1).join("\n"), b.slice(1).join("\n"));
-}
-
 export class DefinitelyTypedRunner extends ExternalCompileRunnerBase {
     readonly testDir = "../DefinitelyTyped/types/";
     override workingDirectory = this.testDir;
@@ -304,31 +130,3 @@ Standard error:
 ${result.stderr.toString().replace(/\r\n/g, "\n")}`;
     }
 }
-
-/**
- * Split an array into multiple arrays whenever `isStart` returns true.
- * @example
- * splitBy([1,2,3,4,5,6], isOdd)
- * ==> [[1, 2], [3, 4], [5, 6]]
- * where
- * const isOdd = n => !!(n % 2)
- */
-function splitBy<T>(xs: T[], isStart: (x: T) => boolean): T[][] {
-    const result = [];
-    let group: T[] = [];
-    for (const x of xs) {
-        if (isStart(x)) {
-            if (group.length) {
-                result.push(group);
-            }
-            group = [x];
-        }
-        else {
-            group.push(x);
-        }
-    }
-    if (group.length) {
-        result.push(group);
-    }
-    return result;
-}

src/testRunner/runner.ts

@@ -3,7 +3,6 @@ import {
     CompilerBaselineRunner,
     CompilerTestType,
     DefinitelyTypedRunner,
-    DockerfileRunner,
     FourSlashRunner,
     GeneratedFourslashRunner,
     IO,
@@ -14,7 +13,6 @@ import {
     setShards,
     Test262BaselineRunner,
     TestRunnerKind,
-    UserCodeRunner,
 } from "./_namespaces/Harness";
 import * as project from "./_namespaces/project";
 import * as RWC from "./_namespaces/RWC";
@@ -80,12 +78,8 @@ export function createRunner(kind: TestRunnerKind): RunnerBase {
             return new RWC.RWCRunner();
         case "test262":
             return new Test262BaselineRunner();
-        case "user":
-            return new UserCodeRunner();
         case "dt":
             return new DefinitelyTypedRunner();
-        case "docker":
-            return new DockerfileRunner();
     }
     return ts.Debug.fail(`Unknown runner kind ${kind}`);
 }
@@ -223,15 +217,9 @@ function handleTestConfig() {
                     case "test262":
                         runners.push(new Test262BaselineRunner());
                         break;
-                    case "user":
-                        runners.push(new UserCodeRunner());
-                        break;
                     case "dt":
                         runners.push(new DefinitelyTypedRunner());
                         break;
-                    case "docker":
-                        runners.push(new DockerfileRunner());
-                        break;
                 }
             }
         }
@@ -250,12 +238,6 @@ function handleTestConfig() {
         runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.ShimsWithPreprocess));
         runners.push(new FourSlashRunner(FourSlash.FourSlashTestType.Server));
         // runners.push(new GeneratedFourslashRunner());
-
-        // CRON-only tests
-        if (process.env.TRAVIS_EVENT_TYPE === "cron") {
-            runners.push(new UserCodeRunner());
-            runners.push(new DockerfileRunner());
-        }
     }
     if (runUnitTests === undefined) {
         runUnitTests = runners.length !== 1; // Don't run unit tests when running only one runner if unit tests were not explicitly asked for