From 680714f5c96552173403df1b70484d205fec4c0f Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Nov 2025 10:35:54 +0100 Subject: [PATCH 1/3] Add docker layer caching for docker builds --- .github/workflows/build-api-lambda.yml | 16 ++++++++++++++-- .../build-link-index-updater-lambda.yml | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-api-lambda.yml b/.github/workflows/build-api-lambda.yml index 069a79ccd..656ee6600 100644 --- a/.github/workflows/build-api-lambda.yml +++ b/.github/workflows/build-api-lambda.yml @@ -24,9 +24,21 @@ jobs: - uses: actions/checkout@v6 with: ref: ${{ inputs.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Amazon Linux 2023 build - run: | - docker build . -t api-lambda:latest -f src/api/Elastic.Documentation.Api.Lambda/Dockerfile + uses: docker/build-push-action@v6 + with: + context: . + file: src/api/Elastic.Documentation.Api.Lambda/Dockerfile + push: false + load: true + tags: api-lambda:latest + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Get bootstrap binary run: | docker cp $(docker create --name tc api-lambda:latest):/app/.artifacts/publish ./.artifacts && docker rm tc diff --git a/.github/workflows/build-link-index-updater-lambda.yml b/.github/workflows/build-link-index-updater-lambda.yml index 89fa3f543..80cd14575 100644 --- a/.github/workflows/build-link-index-updater-lambda.yml +++ b/.github/workflows/build-link-index-updater-lambda.yml @@ -21,9 +21,21 @@ jobs: - uses: actions/checkout@v6 with: ref: ${{ inputs.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Amazon Linux 2023 build - run: | - docker build . -t publish-links-index:latest -f src/infra/docs-lambda-index-publisher/lambda.DockerFile + uses: docker/build-push-action@v6 + with: + context: . + file: src/infra/docs-lambda-index-publisher/lambda.DockerFile + push: false + load: true + tags: publish-links-index:latest + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Get bootstrap binary run: | docker cp $(docker create --name tc publish-links-index:latest):/app/.artifacts/publish ./.artifacts && docker rm tc From c3a98ba1f2cb50c822a34768af9fad1f126b9272 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Nov 2025 11:14:36 +0100 Subject: [PATCH 2/3] Further optimization --- .../Dockerfile | 55 ++++++++++--------- .../lambda.DockerFile | 55 ++++++++++--------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/src/api/Elastic.Documentation.Api.Lambda/Dockerfile b/src/api/Elastic.Documentation.Api.Lambda/Dockerfile index 0153d0aeb..5a2ebee85 100644 --- a/src/api/Elastic.Documentation.Api.Lambda/Dockerfile +++ b/src/api/Elastic.Documentation.Api.Lambda/Dockerfile @@ -5,33 +5,36 @@ ARG TARGETOS WORKDIR /app -RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc - -#RUN curl -o /etc/yum.repos.d/microsoft-prod.repo https://packages.microsoft.com/config/fedora/41/prod.repo - -RUN dnf update -y -#RUN dnf install -y dotnet-sdk-10.0 -RUN dnf install -y npm -RUN dnf install -y git -RUN dnf install -y clang - -## temporary see: https://github.com/amazonlinux/amazon-linux-2023/issues/1025 -RUN dnf install -y tar -RUN dnf install -y findutils -RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh -RUN chmod +x ./dotnet-install.sh -RUN ./dotnet-install.sh -ENV PATH="$PATH:/root/.dotnet" -ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 -# /end temporary +# Set environment variables early (no layer cost) +ENV PATH="$PATH:/root/.dotnet" \ + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \ + DOTNET_NOLOGO=true \ + DOTNET_CLI_TELEMETRY_OPTOUT=true +# Install all system dependencies in a single layer +RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + dnf update -y && \ + dnf install -y npm git clang tar findutils && \ + dnf clean all + +# Install .NET SDK based on global.json version +COPY global.json . +RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh && \ + chmod +x ./dotnet-install.sh && \ + ./dotnet-install.sh --jsonfile ./global.json && \ + rm dotnet-install.sh + +# Copy solution/project files for dependency restoration +# .dockerignore already excludes bin/, obj/, node_modules/, etc. +# This includes all .csproj, .fsproj, .props, and source files +# dotnet restore only reads project files, so having source files here is fine COPY . . -ENV DOTNET_NOLOGO=true \ - DOTNET_CLI_TELEMETRY_OPTOUT=true - -RUN arch=$TARGETARCH \ - && if [ "$arch" = "amd64" ]; then arch="x64"; fi \ - && echo $TARGETOS-$arch > /tmp/rid +# Restore with cache mount for NuGet packages +# This layer is cached until any .csproj/.props file changes +RUN --mount=type=cache,target=/root/.nuget/packages \ + dotnet restore src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -RUN dotnet publish src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -c Release +# Build and publish +RUN --mount=type=cache,target=/root/.nuget/packages \ + dotnet publish src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -c Release --no-restore diff --git a/src/infra/docs-lambda-index-publisher/lambda.DockerFile b/src/infra/docs-lambda-index-publisher/lambda.DockerFile index 5fc45421c..5a5c35e71 100644 --- a/src/infra/docs-lambda-index-publisher/lambda.DockerFile +++ b/src/infra/docs-lambda-index-publisher/lambda.DockerFile @@ -5,33 +5,36 @@ ARG TARGETOS WORKDIR /app -RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc - -#RUN curl -o /etc/yum.repos.d/microsoft-prod.repo https://packages.microsoft.com/config/fedora/41/prod.repo - -RUN dnf update -y -#RUN dnf install -y dotnet-sdk-10.0 -RUN dnf install -y npm -RUN dnf install -y git -RUN dnf install -y clang - -## temporary see: https://github.com/amazonlinux/amazon-linux-2023/issues/1025 -RUN dnf install -y tar -RUN dnf install -y findutils -RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh -RUN chmod +x ./dotnet-install.sh -RUN ./dotnet-install.sh -ENV PATH="$PATH:/root/.dotnet" -ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 -# /end temporary +# Set environment variables early (no layer cost) +ENV PATH="$PATH:/root/.dotnet" \ + DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \ + DOTNET_NOLOGO=true \ + DOTNET_CLI_TELEMETRY_OPTOUT=true +# Install all system dependencies in a single layer +RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ + dnf update -y && \ + dnf install -y npm git clang tar findutils && \ + dnf clean all + +# Install .NET SDK based on global.json version +COPY global.json . +RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh && \ + chmod +x ./dotnet-install.sh && \ + ./dotnet-install.sh --jsonfile ./global.json && \ + rm dotnet-install.sh + +# Copy solution/project files for dependency restoration +# .dockerignore already excludes bin/, obj/, node_modules/, etc. +# This includes all .csproj, .fsproj, .props, and source files +# dotnet restore only reads project files, so having source files here is fine COPY . . -ENV DOTNET_NOLOGO=true \ - DOTNET_CLI_TELEMETRY_OPTOUT=true - -RUN arch=$TARGETARCH \ - && if [ "$arch" = "amd64" ]; then arch="x64"; fi \ - && echo $TARGETOS-$arch > /tmp/rid +# Restore with cache mount for NuGet packages +# This layer is cached until any .csproj/.props file changes +RUN --mount=type=cache,target=/root/.nuget/packages \ + dotnet restore src/infra/docs-lambda-index-publisher -r linux-x64 -RUN dotnet publish src/infra/docs-lambda-index-publisher -r linux-x64 -c Release +# Build and publish +RUN --mount=type=cache,target=/root/.nuget/packages \ + dotnet publish src/infra/docs-lambda-index-publisher -r linux-x64 -c Release --no-restore From c36c0fb4a1ddedf51a7beb0c77cfdaf808b45778 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Tue, 25 Nov 2025 11:18:36 +0100 Subject: [PATCH 3/3] ok --- src/api/Elastic.Documentation.Api.Lambda/Dockerfile | 9 +++++---- src/infra/docs-lambda-index-publisher/lambda.DockerFile | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/api/Elastic.Documentation.Api.Lambda/Dockerfile b/src/api/Elastic.Documentation.Api.Lambda/Dockerfile index 5a2ebee85..360e567f1 100644 --- a/src/api/Elastic.Documentation.Api.Lambda/Dockerfile +++ b/src/api/Elastic.Documentation.Api.Lambda/Dockerfile @@ -30,11 +30,12 @@ RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh && \ # dotnet restore only reads project files, so having source files here is fine COPY . . -# Restore with cache mount for NuGet packages -# This layer is cached until any .csproj/.props file changes +# Restore dependencies +# Note: We use cache mount for NuGet packages, but don't use --no-restore in publish +# because Native AOT compilation needs the full MSBuild evaluation RUN --mount=type=cache,target=/root/.nuget/packages \ dotnet restore src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -# Build and publish +# Build and publish (Native AOT requires full evaluation, so we don't use --no-restore) RUN --mount=type=cache,target=/root/.nuget/packages \ - dotnet publish src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -c Release --no-restore + dotnet publish src/api/Elastic.Documentation.Api.Lambda -r linux-x64 -c Release diff --git a/src/infra/docs-lambda-index-publisher/lambda.DockerFile b/src/infra/docs-lambda-index-publisher/lambda.DockerFile index 5a5c35e71..8692ba580 100644 --- a/src/infra/docs-lambda-index-publisher/lambda.DockerFile +++ b/src/infra/docs-lambda-index-publisher/lambda.DockerFile @@ -30,11 +30,12 @@ RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh && \ # dotnet restore only reads project files, so having source files here is fine COPY . . -# Restore with cache mount for NuGet packages -# This layer is cached until any .csproj/.props file changes +# Restore dependencies +# Note: We use cache mount for NuGet packages, but don't use --no-restore in publish +# because Native AOT compilation needs the full MSBuild evaluation RUN --mount=type=cache,target=/root/.nuget/packages \ dotnet restore src/infra/docs-lambda-index-publisher -r linux-x64 -# Build and publish +# Build and publish (Native AOT requires full evaluation, so we don't use --no-restore) RUN --mount=type=cache,target=/root/.nuget/packages \ - dotnet publish src/infra/docs-lambda-index-publisher -r linux-x64 -c Release --no-restore + dotnet publish src/infra/docs-lambda-index-publisher -r linux-x64 -c Release