Re-do CI build

Per the plan in whatwg/meta#173 (comment), this revamps the CI build to reduce coupling between the html-build and html repositories. Now, running CI (currently Travis) for the html-build repository produces a whatwg/html-build Docker container, pushed to Docker Hub, which can be given an input directory containing the contents of whatwg/html, and an output directory.

The corresponding changes in whatwg/html are in whatwg/html#5945.

Author: domenic

.travis.yml

@@ -3,11 +3,23 @@ services:
   - docker
 
 script:
+  # Shellcheck
   - shellcheck *.sh
-  - shellcheck ci-deploy/*.sh
-  - cd .. &&
-    git clone https://github.com/whatwg/html.git --depth 2 &&
-    IS_TEST_OF_HTML_BUILD_ITSELF=true bash ./html-build/ci-deploy/outside-container.sh
+  - shellcheck ci-build/*.sh
+
+  # Build the Docker container, tagged as whatwg/html-build
+  - bash ci-build/docker-build.sh
+
+  # Test the Docker container
+  - git clone https://github.com/whatwg/html.git --depth 2
+  - mkdir output
+  - bash ci-build/docker-run.sh html output
+
+  # Tag and publish the Docker container
+  - docker tag whatwg/html-build "whatwg/html-build:$(git rev-list --count HEAD)"
+  - docker tag whatwg/html-build whatwg/html-build:latest
+  - if [[ "$TRAVIS_PULL_REQUEST" = "false" ]]; then docker login --username "$DOCKER_USERNAME" --password "$DOCKER_PASSWORD"; fi
+  - if [[ "$TRAVIS_PULL_REQUEST" = "false" ]]; then docker push whatwg/html-build; fi
 
 branches:
   only:

ci-deploy/Dockerfile renamed to ci-build/Dockerfile

@@ -37,18 +37,6 @@ RUN apt-get install --yes --no-install-recommends ./prince_13.5-1_debian10_amd64
     rm prince_13.5-1_debian10_amd64.deb && \
     echo '@font-face { font-family: serif; src: local("Symbola") }' >> /usr/lib/prince/style/fonts.css
 
-ARG html_build_dir
 ADD . /whatwg/html-build
 
-# Note: we do not ADD /whatwg/html, but instead mount it in outside-container.html, since it
-# contains the deploy_key, and thus should not be part of the image. The image is cached, publicly,
-# on Docker Hub.
-ENV HTML_SOURCE /whatwg/html
-
-ARG travis_pull_request
-ARG is_test_of_html_build_itself
-ENV TRAVIS_PULL_REQUEST=${travis_pull_request}
-ENV IS_TEST_OF_HTML_BUILD_ITSELF=${is_test_of_html_build_itself}
-
-ENV SKIP_BUILD_UPDATE_CHECK=true
-ENTRYPOINT ["bash", "/whatwg/html-build/ci-deploy/inside-container.sh"]
+ENTRYPOINT ["bash", "/whatwg/html-build/ci-build/inside-container.sh"]

ci-build/README.md

@@ -0,0 +1,10 @@
+# HTML Standard CI Build
+
+This directory contains the infrastructure for building and running a Docker container, [whatwg/html-build](https://hub.docker.com/r/whatwg/html-build), which performs a "full" build of the HTML Standard, producing artifacts ready for deployment.
+
+The relevant entrypoints are:
+
+- `docker-build.sh` will build the Docker container
+- `docker-run.sh $INPUT $OUTPUT` will run the Docker container to do such a full build.
+  - `$INPUT` should contain a checkout of the [whatwg/html](https://github.com/whatwg/html) repository
+  - `$OUTPUT` should be an empty directory

ci-build/docker-build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+shopt -s extglob
+
+TMP_DIR=$(mktemp -d)
+
+function main {
+  local here
+  here=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )
+
+  # We want the image to contain:
+  # * All of the important stuff from the top-level (html-build) directory
+  # * But, the Dockerfile from this (ci-build) directory
+  # And in particular it should *not* contain the top-level Dockerfile, non-.pl dotfiles, .git/, and
+  # any html/ and output/ directories that might be hanging around from local testing.
+  cp "$here/Dockerfile" "$TMP_DIR"
+  cd "$here/.."
+  cp -r !(.*|html|output|Dockerfile) "$TMP_DIR"
+  cp .*.pl "$TMP_DIR"
+  cd "$TMP_DIR"
+  trap cleanTemp EXIT
+
+  local docker_hub_repo="whatwg/html-build"
+
+  # Build the Docker image, using Docker Hub as a cache. (This will be fast if nothing has changed
+  # in html-build or its dependencies).
+  docker pull whatwg/wattsi
+  docker pull ptspts/pdfsizeopt
+  docker pull "$docker_hub_repo" || true
+  docker build --cache-from "$docker_hub_repo" --tag "$docker_hub_repo" .
+}
+
+function cleanTemp {
+  rm -rf "$TMP_DIR"
+}
+
+main "$@"

ci-build/docker-run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+shopt -s extglob
+
+HTML_SOURCE=$(realpath "$1")
+HTML_OUTPUT=$(realpath "$2")
+
+docker run --rm --mount "type=bind,source=$HTML_SOURCE,destination=/whatwg/html,readonly=1" \
+                --env "HTML_SOURCE=/whatwg/html" \
+                --mount "type=bind,source=$HTML_OUTPUT,destination=/whatwg/output" \
+                --env "HTML_OUTPUT=/whatwg/output" \
+                whatwg/html-build

ci-build/inside-container.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+cd "$(dirname "$0")/../.."
+
+PDF_SERVE_PORT=8080
+
+SKIP_BUILD_UPDATE_CHECK=true ./html-build/build.sh
+
+echo ""
+echo "Running conformance checker..."
+# the -Xmx1g argument sets the size of the Java heap space to 1 gigabyte
+java -Xmx1g -jar ./vnu.jar --skip-non-html "$HTML_OUTPUT"
+echo ""
+
+# Serve the built output so that Prince can snapshot it
+# The nohup/sleep incantations are necessary because normal & does not work inside Docker:
+# https://stackoverflow.com/q/50211207/3191
+(
+  cd "$HTML_OUTPUT"
+  nohup bash -c "python3 -m http.server $PDF_SERVE_PORT &" && sleep 4
+)
+
+echo ""
+echo "Building PDF..."
+PDF_TMP="$(mktemp --suffix=.pdf)"
+prince --verbose --output "$PDF_TMP" "http://0.0.0.0:$PDF_SERVE_PORT/"
+
+echo ""
+echo "Optimizing PDF..."
+PATH=/bin/pdfsizeopt:$PATH pdfsizeopt --v=30 "$PDF_TMP" "$HTML_OUTPUT/print.pdf"