diff --git a/src/argv.ts b/src/argv.ts index 4b182c939..8f6f9b3e6 100644 --- a/src/argv.ts +++ b/src/argv.ts @@ -350,4 +350,8 @@ export class Argv { get childPipelineDepth (): number { return this.map.get("childPipelineDepth"); } + + get registry (): boolean { + return this.map.get("registry") ?? false; + } } diff --git a/src/handler.ts b/src/handler.ts index 0b9621a29..b70c95559 100644 --- a/src/handler.ts +++ b/src/handler.ts @@ -69,6 +69,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ Commander.runCsv(parser, writeStreams, argv.listCsvAll); } else if (argv.job.length > 0) { assert(argv.stage === null, "You cannot use --stage when starting individual jobs"); + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); if (argv.needs || argv.onlyNeeds) { @@ -82,6 +85,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } } else if (argv.stage) { + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); const pipelineIid = await state.getPipelineIid(cwd, stateDir); @@ -90,6 +96,9 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ await Commander.runJobsInStage(argv, parser, writeStreams); writeStreams.stderr(chalk`{grey pipeline finished} in {grey ${prettyHrtime(process.hrtime(time))}}\n`); } else { + if (argv.registry) { + await Utils.startDockerRegistry(argv); + } generateGitIgnore(cwd, stateDir); const time = process.hrtime(); await state.incrementPipelineIid(cwd, stateDir); @@ -101,5 +110,8 @@ export async function handler (args: any, writeStreams: WriteStreams, jobs: Job[ } writeStreams.flush(); + if (argv.registry) { + await Utils.stopDockerRegistry(argv.containerExecutable); + } return cleanupJobResources(jobs); } diff --git a/src/index.ts b/src/index.ts index 523f2ccd9..63e3e4fdc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -323,6 +323,11 @@ process.on("SIGUSR2", async () => await cleanupJobResources(jobs)); default: true, description: "Enables color", }) + .option("registry", { + type: "boolean", + requiresArg: false, + description: "Start a local docker registry and configure gitlab-ci-local containers to use that by default", + }) .completion("completion", false, (current: string, yargsArgv: any, completionFilter: any, done: (completions: string[]) => any) => { try { if (current.startsWith("-")) { diff --git a/src/job.ts b/src/job.ts index 0d8ce22d2..619116f62 100644 --- a/src/job.ts +++ b/src/job.ts @@ -329,7 +329,7 @@ export class Job { predefinedVariables["CI_NODE_INDEX"] = `${opt.nodeIndex}`; } predefinedVariables["CI_NODE_TOTAL"] = `${opt.nodesTotal}`; - predefinedVariables["CI_REGISTRY"] = `local-registry.${this.gitData.remote.host}`; + predefinedVariables["CI_REGISTRY"] = predefinedVariables["CI_REGISTRY"] = this.argv.registry ? Utils.gclRegistryPrefix : `local-registry.${this.gitData.remote.host}`; predefinedVariables["CI_REGISTRY_IMAGE"] = `$CI_REGISTRY/${predefinedVariables["CI_PROJECT_PATH"].toLowerCase()}`; return predefinedVariables; } @@ -886,6 +886,11 @@ export class Job { }); } + if (this.argv.registry) { + expanded["CI_REGISTRY_USER"] = expanded["CI_REGISTRY_USER"] ?? `${Utils.gclRegistryPrefix}.user`; + expanded["CI_REGISTRY_PASSWORD"] = expanded["CI_REGISTRY_PASSWORD"] ?? `${Utils.gclRegistryPrefix}.password`; + } + this.refreshLongRunningSilentTimeout(writeStreams); if (imageName && !this._containerId) { @@ -954,6 +959,12 @@ export class Job { dockerCmd += `--network ${this._serviceNetworkId} --network-alias build `; } + if (this.argv.registry) { + dockerCmd += `--network ${Utils.gclRegistryPrefix}.net `; + dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/containers/certs.d:ro `; + dockerCmd += `--volume ${Utils.gclRegistryPrefix}.certs:/etc/docker/certs.d:ro `; + } + dockerCmd += `--volume ${buildVolumeName}:${this.ciProjectDir} `; dockerCmd += `--volume ${tmpVolumeName}:${this.fileVariablesDir} `; dockerCmd += `--workdir ${this.ciProjectDir} `; diff --git a/src/utils.ts b/src/utils.ts index a8ab4f180..c4e5d41ef 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -5,7 +5,7 @@ import {Job, JobRule} from "./job.js"; import fs from "fs-extra"; import checksum from "checksum"; import base64url from "base64url"; -import execa from "execa"; +import execa, {ExecaError} from "execa"; import assert from "assert"; import {CICDVariable} from "./variables-from-files.js"; import {GitData, GitSchema} from "./git-data.js"; @@ -438,6 +438,101 @@ export class Utils { throw new Error(`Unhandled case ${param}`); } + static async dockerVolumeFileExists (containerExecutable: string, path: string, volume: string): Promise { + try { + await Utils.spawn([containerExecutable, "run", "--rm", "-v", `${volume}:/mnt/vol`, "alpine", "ls", `/mnt/vol/${path}`]); + return true; + } catch { + return false; + } + } + + static gclRegistryPrefix: string = "registry.gcl.local"; + static async startDockerRegistry (argv: Argv): Promise { + const gclRegistryCertVol = `${this.gclRegistryPrefix}.certs`; + const gclRegistryDataVol = `${this.gclRegistryPrefix}.data`; + const gclRegistryNet = `${this.gclRegistryPrefix}.net`; + + // create cert volume + try { + await Utils.spawn(`${argv.containerExecutable} volume create ${gclRegistryCertVol}`.split(" ")); + } catch (err) { + if (err instanceof Error && !err.message.endsWith("already exists")) + throw err; + } + + // create self-signed cert/key files for https support + if (!await this.dockerVolumeFileExists(argv.containerExecutable, `${this.gclRegistryPrefix}.crt`, gclRegistryCertVol)) { + const opensslArgs = [ + "req", "-newkey", "rsa:4096", "-nodes", "-sha256", + "-keyout", `/certs/${this.gclRegistryPrefix}.key`, + "-x509", "-days", "365", + "-out", `/certs/${this.gclRegistryPrefix}.crt`, + "-subj", `/CN=${this.gclRegistryPrefix}`, + "-addext", `subjectAltName=DNS:${this.gclRegistryPrefix}`, + ]; + const generateCertsInPlace = [ + argv.containerExecutable, "run", "--rm", "-v", `${gclRegistryCertVol}:/certs`, "--entrypoint", "sh", "alpine/openssl", "-c", + [ + "openssl", ...opensslArgs, + "&&", "mkdir", "-p", `/certs/${this.gclRegistryPrefix}`, + "&&", "cp", `/certs/${this.gclRegistryPrefix}.crt`, `/certs/${this.gclRegistryPrefix}/ca.crt`, + ].join(" "), + ]; + await Utils.spawn(generateCertsInPlace); + } + + // create data volume + try { + await Utils.spawn([argv.containerExecutable, "volume", "create", gclRegistryDataVol]); + } catch (err) { + if (err instanceof Error && !err.message.endsWith("already exists")) + throw err; + } + + // create network + try { + await Utils.spawn([argv.containerExecutable, "network", "create", gclRegistryNet]); + } catch (err) { + if (err instanceof Error && !err.message.includes("already exists")) + throw err; + } + + await Utils.spawn([argv.containerExecutable, "rm", "-f", this.gclRegistryPrefix]); + await Utils.spawn([ + argv.containerExecutable, "run", "-d", "--name", this.gclRegistryPrefix, + "--network", gclRegistryNet, + "--volume", `${gclRegistryDataVol}:/var/lib/registry`, + "--volume", `${gclRegistryCertVol}:/certs:ro`, + "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:443", + "-e", `REGISTRY_HTTP_TLS_CERTIFICATE=/certs/${this.gclRegistryPrefix}.crt`, + "-e", `REGISTRY_HTTP_TLS_KEY=/certs/${this.gclRegistryPrefix}.key`, + "registry", + ]); + + try { + await execa(argv.containerExecutable, [ + "run", "--rm", + "--network", gclRegistryNet, + "--entrypoint", "sh", + "curlimages/curl", + "-c", `until [ "$(curl -s -o /dev/null -k -w "%{http_code}" https://${this.gclRegistryPrefix}:443)" = "200" ]; do sleep 1; done;`, + ], { + timeout: 4000, + }); + } catch (err) { + await this.stopDockerRegistry(argv.containerExecutable); + if ((err as ExecaError).timedOut) { + throw "local docker registry port check timed out"; + } + throw err; + } + } + + static async stopDockerRegistry (containerExecutable: string): Promise { + await Utils.spawn([containerExecutable, "rm", "-f", this.gclRegistryPrefix]); + } + static async getTrackedFiles (cwd: string): Promise { const lsFilesRes = await Utils.bash("git ls-files --deduplicate", cwd); if (lsFilesRes.exitCode != 0) { diff --git a/tests/test-cases/local-registry/.gitlab-ci.yml b/tests/test-cases/local-registry/.gitlab-ci.yml new file mode 100644 index 000000000..158a0b6ae --- /dev/null +++ b/tests/test-cases/local-registry/.gitlab-ci.yml @@ -0,0 +1,17 @@ +--- +registry-variables: + image: alpine:latest + script: + - echo "CI_REGISTRY=$CI_REGISTRY" + - echo "CI_REGISTRY_USER=$CI_REGISTRY_USER" + - echo "CI_REGISTRY_PASSWORD=$CI_REGISTRY_PASSWORD" + +registry-login-docker: + image: docker:dind + script: + - echo "$CI_REGISTRY_PASSWORD" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + +registry-login-oci: + image: quay.io/podman/stable + script: + - echo "$CI_REGISTRY_PASSWORD" | podman login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY diff --git a/tests/test-cases/local-registry/integration.test.ts b/tests/test-cases/local-registry/integration.test.ts new file mode 100644 index 000000000..7f850ed43 --- /dev/null +++ b/tests/test-cases/local-registry/integration.test.ts @@ -0,0 +1,54 @@ +import {WriteStreamsMock} from "../../../src/write-streams.js"; +import {handler} from "../../../src/handler.js"; +import {Utils} from "../../../src/utils.js"; +import chalk from "chalk"; + +test("local-registry ci variables", async () => { + const writeStreams = new WriteStreamsMock; + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-variables"], + registry: true, + }, writeStreams); + + const expected = [ + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY=${Utils.gclRegistryPrefix}`, + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY_USER=${Utils.gclRegistryPrefix}.user`, + chalk`{blueBright registry-variables} {greenBright >} CI_REGISTRY_PASSWORD=${Utils.gclRegistryPrefix}.password`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); + +test("local-registry login ", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-login-docker"], + registry: true, + }, writeStreams); + + + const expected = [ + chalk`{blueBright registry-login-docker} {greenBright >} Login Succeeded`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +}); + +test("local-registry login ", async () => { + const writeStreams = new WriteStreamsMock(); + await handler({ + cwd: "tests/test-cases/local-registry", + job: ["registry-login-oci"], + registry: true, + privileged: true, + }, writeStreams); + + + const expected = [ + chalk`{blueBright registry-login-oci} {greenBright >} Login Succeeded!`, + ]; + + expect(writeStreams.stdoutLines).toEqual(expect.arrayContaining(expected)); +});