Run separate web server, registry watcher and build servers in docker-compose

Author: Nemo157

docker-compose.yml

@@ -4,33 +4,85 @@ services:
         build:
             context: .
             dockerfile: ./dockerfiles/Dockerfile
+            target: web-server
         platform: "linux/amd64"
         depends_on:
             - db
             - s3
         ports:
-            - "3000:3000"
-        # for metrics
-        expose: ["3000"]
+            - "3000:80"
+        env_file:
+            - docker.env
+            - .env
+        healthcheck:
+            test: ["CMD", "curl", "--silent", "--fail", "localhost:80"]
+            interval: 10s
+            timeout: 5s
+            retries: 10
+
+    registry-watcher:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: registry-watcher
+        platform: "linux/amd64"
+        depends_on:
+            - db
         volumes:
-            - "/var/run/docker.sock:/var/run/docker.sock"
-            - ".rustwide-docker:/opt/docsrs/rustwide"
             - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
-        environment:
-            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
-            DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
-            DOCSRS_STORAGE_BACKEND: s3
-            S3_ENDPOINT: http://s3:9000
-            AWS_ACCESS_KEY_ID: cratesfyi
-            AWS_SECRET_ACCESS_KEY: secret_key
         env_file:
+            - docker.env
             - .env
         healthcheck:
             test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"]
             interval: 10s
             timeout: 5s
             retries: 10
 
+    builder-a:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: build-server
+        depends_on:
+            - db
+            - s3
+        volumes:
+            - ".rustwide-docker/a:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+        env_file:
+            - docker.env
+            - .env
+
+    builder-b:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: build-server
+        depends_on:
+            - db
+            - s3
+        volumes:
+            - ".rustwide-docker/b:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+        env_file:
+            - docker.env
+            - .env
+
+    cli:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: cli
+        depends_on:
+            - db
+            - s3
+        volumes:
+            - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
+        env_file:
+            - docker.env
+            - .env
+
     db:
         build:
             context: ./dockerfiles

docker.env

@@ -0,0 +1,24 @@
+# general environment variables shared between all docker compose services
+# you can override these in a .env file if needed
+
+export RUST_BACKTRACE=true
+
+export DOCSRS_PREFIX=/opt/docsrs/prefix
+
+export DOCSRS_DATABASE_URL=postgresql://cratesfyi:password@db
+export DOCSRS_MIN_POOL_SIZE=2
+export DOCSRS_MAX_POOL_SIZE=10
+
+export DOCSRS_STORAGE_BACKEND=s3
+
+export S3_ENDPOINT=http://s3:9000
+export AWS_ACCESS_KEY_ID=cratesfyi
+export AWS_SECRET_ACCESS_KEY=secret_key
+
+export DOCSRS_RENDER_THREADS=2
+
+export DOCSRS_RUSTWIDE_WORKSPACE=/opt/docsrs/rustwide
+export DOCSRS_DOCKER=true
+export DOCSRS_DOCKER_IMAGE=ghcr.io/rust-lang/crates-build-env/linux-micro
+export DOCSRS_BUILD_CPU_LIMIT=2
+export DOCSRS_INCLUDE_DEFAULT_TARGETS=false

dockerfiles/Dockerfile

@@ -77,6 +77,67 @@ WORKDIR /srv/docsrs
 # Tini is a small init binary to properly handle signals
 CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "start-web-server", "0.0.0.0:80"]
 
+########################
+#  Build server stage  #
+########################
+
+FROM ubuntu:22.04 AS build-server
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        ca-certificates \
+        tini \
+        docker.io \
+        build-essential \
+        gcc \
+        pkg-config \
+        libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /build/target/release/cratesfyi /usr/local/bin
+
+WORKDIR /srv/docsrs
+# Tini is a small init binary to properly handle signals
+CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "start-build-server"]
+
+############################
+#  Registry watcher stage  #
+############################
+
+FROM ubuntu:22.04 AS registry-watcher
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        ca-certificates \
+        tini \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /build/target/release/cratesfyi /usr/local/bin
+
+WORKDIR /srv/docsrs
+# Tini is a small init binary to properly handle signals
+CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--", "start-registry-watcher", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled"]
+
+###############
+#  CLI stage  #
+###############
+
+FROM ubuntu:22.04 AS cli
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+        ca-certificates \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /build/target/release/cratesfyi /usr/local/bin
+
+ENTRYPOINT ["/usr/local/bin/cratesfyi"]
+
 ##################
 #  Output stage  #
 ##################