Skip to content

Commit 6233161

Browse files
domenicdomenic
authored
Improvements to the CI Docker build
This is a series of improvements that will help pave the way for a better CI build/deploy architecture in the future, potentially based on GitHub Actions. In particular: * Stop grabbing pdfsizeopt using git from outside the container. Instead, get it as part of the Dockerfile along with other dependencies. Closes #221. * Avoid the multi-step PDF generation process, where we deploy to whatwg.org, point Prince at that, and then deploy the generated print.pdf to whatwg.org. Instead, we serve the built output inside the Docker container, use that to generate print.pdf, and deploy everything to whatwg.org in a single step. * Update Prince from v11.3 to v13.5. * Download only the .jar file for the validator, instead of the full package; we can let Docker handle installing Java.
1 parent bc5e00f commit 6233161

File tree

3 files changed

+106
-98
lines changed

3 files changed

+106
-98
lines changed

‎ci-deploy/Dockerfile‎

Lines changed: 25 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,47 +3,44 @@
33
FROM debian:stable
44

55
RUN apt-get update && \
6-
apt-get install -y ca-certificates curl rsync git unzip \
7-
default-jre \
8-
libfontconfig1 libgomp1 libxml2 \
9-
python3 python3-pip \
6+
apt-get install --yes --no-install-recommends \
7+
ca-certificates curl rsync git \
8+
default-jre \
9+
python3 python3-pip \
1010
fonts-dejavu fonts-droid-fallback fonts-liberation fonts-symbola fonts-unfonts-core
1111

1212
# Dependency lines above are:
1313
# - General
1414
# - validator
15-
# - Prince
1615
# - Highlighter
17-
# - fonts
16+
# - fonts, for when Prince renders to PDF
1817

18+
COPY --from=whatwg/wattsi:latest /whatwg/wattsi/bin/wattsi /bin/wattsi
19+
COPY --from=ptspts/pdfsizeopt:latest /bin /bin/pdfsizeopt
1920
RUN pip3 install bs-highlighter
2021

21-
# Dependecies of prince_11.3-1_debian8.0_amd64.deb (not used) are libc6 libcurl3 libfontconfig1
22-
# libfreetype6 libgif4 libgomp1 libjpeg62-turbo libpng12-0 libssl1.0.0 libtiff5 libxml2 zlib1g.
23-
# Above we install only the subset that's needed to make Prince work.
24-
25-
ADD https://www.princexml.com/download/prince-11.3-linux-generic-x86_64.tar.gz /whatwg/prince.tar.gz
22+
# The DockerHub container for pdfsizeopt has some useful auxiliary binaries it depends on, but the
23+
# main binary is outdated and needs updating directly from GitHub:
24+
# TODO: consolidate these two lines when https://github.com/moby/buildkit/pull/1492 gets released
25+
# (see https://github.com/moby/moby/issues/34819).
26+
ADD https://github.com/pts/pdfsizeopt/blob/master/pdfsizeopt.single?raw=true /bin/pdfsizeopt/pdfsizeopt
27+
RUN chmod a+rwx /bin/pdfsizeopt/pdfsizeopt
2628

27-
RUN cd /whatwg && \
28-
tar xzf prince.tar.gz && \
29-
echo /whatwg/prince | /whatwg/prince-11.3-linux-generic-x86_64/install.sh && \
30-
echo '@font-face { font-family: serif; src: local("Symbola") }' >> /whatwg/prince/lib/prince/style/fonts.css && \
31-
rm -rf /whatwg/prince.tar.gz /whatwg/prince-11.3-linux-generic-x86_64
32-
ENV PATH="/whatwg/prince/bin:${PATH}"
29+
# The DockerHub container for the validator only contains the server version, so we get the .jar
30+
# from GitHub:
31+
ADD https://github.com/validator/validator/releases/download/jar/vnu.jar /whatwg/
3332

34-
ADD pdfsizeopt/pdfsizeopt.single /whatwg/pdfsizeopt/bin/pdfsizeopt
35-
ADD https://github.com/pts/pdfsizeopt/releases/download/2017-01-24/pdfsizeopt_libexec_linux-v3.tar.gz /whatwg/pdfsizeopt_libexec.tar.gz
36-
RUN cd /whatwg && \
37-
tar xzf pdfsizeopt_libexec.tar.gz && \
38-
mv pdfsizeopt_libexec/* pdfsizeopt/bin/ && \
39-
rm -rf pdfsizeopt_libexec.tar.gz pdfsizeopt_libexec
40-
ENV PATH="/whatwg/pdfsizeopt/bin:${PATH}"
41-
42-
COPY --from=whatwg/wattsi:latest /whatwg/wattsi/bin/wattsi /bin/wattsi
33+
# Trying to copy Prince from its DockerHub container like the others does not work; it has too many
34+
# shared library dependencies. So we get the .deb file from the source:
35+
ADD https://www.princexml.com/download/prince_13.5-1_debian10_amd64.deb .
36+
RUN apt-get install --yes --no-install-recommends ./prince_13.5-1_debian10_amd64.deb && \
37+
rm prince_13.5-1_debian10_amd64.deb && \
38+
echo '@font-face { font-family: serif; src: local("Symbola") }' >> /usr/lib/prince/style/fonts.css
4339

44-
ADD html-build /whatwg/html-build
40+
ARG html_build_dir
41+
ADD . /whatwg/html-build
4542

46-
# Note: we do not ADD /whatwg/html, but instead mount it using --volume in .travis.yml, since it
43+
# Note: we do not ADD /whatwg/html, but instead mount it in outside-container.html, since it
4744
# contains the deploy_key, and thus should not be part of the image. The image is cached, publicly,
4845
# on Docker Hub.
4946
ENV HTML_SOURCE /whatwg/html

‎ci-deploy/inside-container.sh‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -o nounset
44
set -o pipefail
55
cd "$(dirname "$0")/../.."
66

7-
PDF_SOURCE_URL="https://html.spec.whatwg.org/"
7+
PDF_SERVE_PORT=8080
88
WEB_ROOT="html.spec.whatwg.org"
99
COMMITS_DIR="commit-snapshots"
1010
REVIEW_DIR="review-drafts"
@@ -26,13 +26,28 @@ IS_TEST_OF_HTML_BUILD_ITSELF=${IS_TEST_OF_HTML_BUILD_ITSELF:-false}
2626

2727
# Conformance-check the result
2828
echo ""
29-
echo "Downloading and running conformance checker..."
30-
curl --retry 2 --remote-name --fail --location https://github.com/validator/validator/releases/download/linux/vnu.linux.zip
31-
unzip vnu.linux.zip
29+
echo "Running conformance checker..."
3230
# the -Xmx1g argument sets the size of the Java heap space to 1 gigabyte
33-
./vnu-runtime-image/bin/java -Xmx1g -m vnu/nu.validator.client.SimpleCommandLineValidator --skip-non-html "$HTML_OUTPUT"
31+
java -Xmx1g -jar ./vnu.jar --skip-non-html "$HTML_OUTPUT"
3432
echo ""
3533

34+
# Serve the built output so that Prince can snapshot it
35+
# The nohup/sleep incantations are necessary because normal & does not work inside Docker:
36+
# https://stackoverflow.com/q/50211207/3191
37+
(
38+
cd "$HTML_OUTPUT"
39+
nohup bash -c "python3 -m http.server $PDF_SERVE_PORT &" && sleep 4
40+
)
41+
42+
echo ""
43+
echo "Building PDF..."
44+
PDF_TMP="$(mktemp --suffix=.pdf)"
45+
prince --verbose --output "$PDF_TMP" "http://0.0.0.0:$PDF_SERVE_PORT/"
46+
47+
echo ""
48+
echo "Optimizing PDF..."
49+
PATH=/bin/pdfsizeopt:$PATH pdfsizeopt --v=30 "$PDF_TMP" "$HTML_OUTPUT/print.pdf"
50+
3651
if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
3752
echo "Skipping deploy for non-master"
3853
exit 0
@@ -54,7 +69,6 @@ echo "Deploying build output..."
5469
rsync --rsh="ssh -o UserKnownHostsFile=known_hosts" \
5570
--archive --chmod=D755,F644 --compress --verbose \
5671
--delete --exclude="$COMMITS_DIR" --exclude="$REVIEW_DIR" \
57-
--exclude=print.pdf \
5872
"$HTML_OUTPUT/" "deploy@$SERVER:/var/www/$WEB_ROOT"
5973

6074
# Now sync a commit snapshot and a review draft, if any
@@ -66,20 +80,5 @@ rsync --rsh="ssh -o UserKnownHostsFile=known_hosts" \
6680
--archive --chmod=D755,F644 --compress --verbose \
6781
"$HTML_OUTPUT/$COMMITS_DIR" "$HTML_OUTPUT/$REVIEW_DIR" "deploy@$SERVER:/var/www/$WEB_ROOT"
6882

69-
echo ""
70-
echo "Building PDF..."
71-
PDF_TMP="$(mktemp --suffix=.pdf)"
72-
prince --verbose --output "$PDF_TMP" "$PDF_SOURCE_URL"
73-
74-
echo ""
75-
echo "Optimizing PDF..."
76-
pdfsizeopt --v=40 "$PDF_TMP" "$HTML_OUTPUT/print.pdf"
77-
78-
echo ""
79-
echo "Deploying PDF..."
80-
rsync --rsh="ssh -o UserKnownHostsFile=known_hosts" \
81-
--archive --compress --verbose \
82-
"$HTML_OUTPUT/print.pdf" "deploy@$SERVER:/var/www/$WEB_ROOT/print.pdf"
83-
8483
echo ""
8584
echo "All done!"

‎ci-deploy/outside-container.sh‎

Lines changed: 61 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,54 +2,66 @@
22
set -o errexit
33
set -o nounset
44
set -o pipefail
5+
shopt -s extglob
56

6-
HERE=$(dirname "$0")
7-
cd "$HERE/../.."
8-
9-
DOCKER_USERNAME="domenicdenicola"
10-
DOCKER_HUB_REPO="whatwg/html-deploy"
11-
12-
# Set from the outside:
13-
TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}
14-
IS_TEST_OF_HTML_BUILD_ITSELF=${IS_TEST_OF_HTML_BUILD_ITSELF:-false}
15-
16-
# When not running pull request builds:
17-
# - DOCKER_PASSWORD is set from the outside
18-
# - ENCRYPTION_LABEL is set from the outside
19-
20-
git clone --depth 1 https://github.com/pts/pdfsizeopt.git pdfsizeopt
21-
22-
# Copy the Docker-related stuff into the working (grandparent) directory.
23-
cp "$HERE"/{.dockerignore,Dockerfile} .
24-
25-
# Build the Docker image, using Docker Hub as a cache. (This will be fast if nothing has changed
26-
# in wattsi or html-build).
27-
docker pull "$DOCKER_HUB_REPO:latest"
28-
docker build --cache-from "$DOCKER_HUB_REPO:latest" \
29-
--tag "$DOCKER_HUB_REPO:latest" \
30-
--build-arg "travis_pull_request=$TRAVIS_PULL_REQUEST" \
31-
--build-arg "is_test_of_html_build_itself=$IS_TEST_OF_HTML_BUILD_ITSELF" \
32-
.
33-
if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$IS_TEST_OF_HTML_BUILD_ITSELF" == "false" ]]; then
34-
# Decrypt the deploy key from this script's location into the html/ directory, since that's the
35-
# directory that will be shared with the container (but not built into the image).
36-
ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
37-
ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv"
38-
ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
39-
ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
40-
openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" \
41-
-in "$HERE/deploy-key.enc" -out html/deploy-key -d
42-
fi
43-
44-
# Run the inside-container.sh script, with the html/ directory mounted inside the container.
45-
echo ""
46-
docker run --volume "$(pwd)/html":/whatwg/html "$DOCKER_HUB_REPO:latest"
47-
48-
if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$IS_TEST_OF_HTML_BUILD_ITSELF" == "false" ]]; then
49-
# If the build succeeded and we got here, upload the Docker image to Docker Hub, so that future runs
50-
# can use it as a cache.
7+
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
8+
TMP_DIR=$(mktemp -d)
9+
10+
function main {
11+
cp "$HERE/Dockerfile" "$TMP_DIR"
12+
cd "$HERE/.."
13+
cp -r !(.*|html|Dockerfile) "$TMP_DIR"
14+
cp .*.pl "$TMP_DIR"
15+
cd "$TMP_DIR"
16+
trap cleanTemp EXIT
17+
18+
DOCKER_USERNAME="domenicdenicola"
19+
DOCKER_HUB_REPO="whatwg/html-deploy"
20+
21+
# Set from the outside:
22+
TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST:-false}
23+
IS_TEST_OF_HTML_BUILD_ITSELF=${IS_TEST_OF_HTML_BUILD_ITSELF:-false}
24+
25+
# When not running pull request builds:
26+
# - DOCKER_PASSWORD is set from the outside
27+
# - ENCRYPTION_LABEL is set from the outside
28+
29+
# Build the Docker image, using Docker Hub as a cache. (This will be fast if nothing has changed
30+
# in wattsi or html-build).
31+
docker build --cache-from "$DOCKER_HUB_REPO:latest" \
32+
--tag "$DOCKER_HUB_REPO:latest" \
33+
--build-arg "html_build_dir=$TMP_DIR" \
34+
--build-arg "travis_pull_request=$TRAVIS_PULL_REQUEST" \
35+
--build-arg "is_test_of_html_build_itself=$IS_TEST_OF_HTML_BUILD_ITSELF" \
36+
.
37+
if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$IS_TEST_OF_HTML_BUILD_ITSELF" == "false" ]]; then
38+
# Decrypt the deploy key from this script's location into the html/ directory, since that's the
39+
# directory that will be shared with the container (but not built into the image).
40+
ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
41+
ENCRYPTED_IV_VAR="encrypted_${ENCRYPTION_LABEL}_iv"
42+
ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
43+
ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
44+
openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" \
45+
-in "$HERE/deploy-key.enc" -out html/deploy-key -d
46+
fi
47+
48+
# Run the inside-container.sh script, with the html/ directory mounted inside the container.
5149
echo ""
52-
docker tag "$DOCKER_HUB_REPO:latest" "$DOCKER_HUB_REPO:$TRAVIS_BUILD_NUMBER" &&
53-
docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
54-
docker push "$DOCKER_HUB_REPO"
55-
fi
50+
cd "$HERE/../.."
51+
docker run --mount "type=bind,source=$(pwd)/html,destination=/whatwg/html,readonly=1" "$DOCKER_HUB_REPO:latest"
52+
53+
if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$IS_TEST_OF_HTML_BUILD_ITSELF" == "false" ]]; then
54+
# If the build succeeded and we got here, upload the Docker image to Docker Hub, so that future runs
55+
# can use it as a cache.
56+
echo ""
57+
docker tag "$DOCKER_HUB_REPO:latest" "$DOCKER_HUB_REPO:$TRAVIS_BUILD_NUMBER" &&
58+
docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD"
59+
docker push "$DOCKER_HUB_REPO"
60+
fi
61+
}
62+
63+
function cleanTemp {
64+
rm -rf "$TMP_DIR"
65+
}
66+
67+
main "$@"

0 commit comments

Comments
 (0)