refactor(ci): extract common AMI build logic into reusable action and add actionlint

jfroche · jfroche · commit 6c863d14f130 · 2025-12-04T16:31:39.000+01:00
Consolidate duplicate AMI build steps across workflows into a shared
composite action. Also introduces actionlint configuration for GitHub
Actions validation for the modified workflows.
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
@@ -0,0 +1,6 @@
+self-hosted-runner:
+  labels:
+    - blacksmith-2vcpu-ubuntu-2404
+    - blacksmith-2vcpu-ubuntu-2404-arm
+    - blacksmith-4vcpu-ubuntu-2404
+    - large-linux-arm
diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml
@@ -0,0 +1,87 @@
+name: Build AMI
+description: Build both stage 1 and stage 2 AMIs
+
+inputs:
+  postgres_version:
+    description: 'PostgreSQL major version (e.g., 15)'
+    required: true
+  region:
+    description: 'AWS region'
+    required: true
+  ami_regions:
+    description: 'AMI regions as JSON array (e.g., ["us-east-1"])'
+    required: true
+  git_sha:
+    description: 'Git SHA for this build'
+    required: true
+
+outputs:
+  stage2_ami_id:
+    description: 'The AMI ID of the stage 2 build'
+    value: ${{ steps.build-stage2.outputs.stage2_ami_id }}
+  postgres_release_version:
+    description: 'The PostgreSQL release version'
+    value: ${{ steps.generate-vars.outputs.version }}
+  execution_id:
+    description: 'The execution ID for this build'
+    value: ${{ steps.set-execution-id.outputs.execution_id }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set execution ID
+      id: set-execution-id
+      shell: bash
+      run: |
+        EXECUTION_ID="${{ github.run_id }}-${{ inputs.postgres_version }}"
+        echo "EXECUTION_ID=$EXECUTION_ID" >> $GITHUB_ENV
+        echo "execution_id=$EXECUTION_ID" >> $GITHUB_OUTPUT
+
+    - name: Generate common-nix.vars.pkr.hcl
+      id: generate-vars
+      shell: bash
+      run: |
+        PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ inputs.postgres_version }}"]' ansible/vars.yml)
+        PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')
+        echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
+        echo "" >> common-nix.vars.pkr.hcl
+        git add -f common-nix.vars.pkr.hcl
+        echo "version=$PG_VERSION" >> $GITHUB_OUTPUT
+
+    - name: Build AMI stage 1
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+      run: |
+        nix run .#build-ami -- stage1 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \
+          -var "region=${{ inputs.region }}" \
+          -var 'ami_regions=${{ inputs.ami_regions }}' \
+          -var-file="development-arm.vars.pkr.hcl" \
+          -var-file="common-nix.vars.pkr.hcl" \
+          amazon-arm64-nix.pkr.hcl
+
+    - name: Build AMI stage 2
+      id: build-stage2
+      shell: bash
+      env:
+        POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }}
+        PACKER_EXECUTION_ID: ${{ env.EXECUTION_ID }}
+        AWS_MAX_ATTEMPTS: 10
+        AWS_RETRY_MODE: adaptive
+      run: |
+        nix run .#build-ami -- stage2 \
+          -var "git-head-version=${{ inputs.git_sha }}" \
+          -var "packer-execution-id=${{ env.EXECUTION_ID }}" \
+          -var "postgres_major_version=${{ inputs.postgres_version }}" \
+          -var-file="development-arm.vars.pkr.hcl" \
+          -var-file="common-nix.vars.pkr.hcl" \
+          -var "postgres-version=${{ env.EXECUTION_ID }}" \
+          -var 'ami_regions=${{ inputs.ami_regions }}' \
+          -var "force-deregister=true" \
+          -var "git_sha=${{ inputs.git_sha }}" \
+          stage2-nix-psql.pkr.hcl
diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml
@@ -27,6 +27,7 @@ jobs:
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
         with:
           ref: ${{ github.event.inputs.branch }}
+
       - name: aws-creds
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -38,56 +39,35 @@ jobs:
       - name: Get current branch SHA
         id: get_sha
         run: |
-          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
+        uses: ./.github/actions/nix-install-ephemeral
         with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
-
-      - name: Set PostgreSQL version environment variable
-        run: |
-          echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV
-          echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      - name: Build AMI stage 1
+          push-to-cache: 'true'
         env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
+          DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
 
-      - name: Build AMI stage 2
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
-          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
+        with:
+          postgres_version: ${{ github.event.inputs.postgres_version }}
+          region: us-east-1
+          ami_regions: '["us-east-1"]'
+          git_sha: ${{ steps.get_sha.outputs.sha }}
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          VERSION="${{ steps.build-ami.outputs.postgres_release_version }}"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          MAJOR_VERSION=${{ github.event.inputs.postgres_version }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -105,7 +85,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
@@ -126,7 +106,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ github.event.inputs.postgres_version }}" \
             manifest-playbook.yml
     
       - name: Upload nix flake revision to s3 prod
@@ -155,10 +135,12 @@ jobs:
       - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids 
 
diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml
@@ -25,18 +25,13 @@ jobs:
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
-        with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+        uses: ./.github/actions/nix-install-ephemeral
 
       - name: Set PostgreSQL versions
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
-          echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
+          VERSIONS=$(nix run nixpkgs#yq -- -r '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          echo "postgres_versions=$VERSIONS" >> "$GITHUB_OUTPUT"
 
   build:
     needs: prepare
@@ -51,6 +46,12 @@ jobs:
     steps:
       - name: Checkout Repo
         uses: supabase/postgres/.github/actions/shared-checkout@HEAD
+        with:
+          push-to-cache: 'true'
+        env:
+          DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }}
+          NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }}
+
       - name: aws-creds
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -60,12 +61,7 @@ jobs:
           role-duration-seconds: 7200
 
       - name: Install nix
-        uses: cachix/install-nix-action@v27
-        with:
-          install_url: https://releases.nixos.org/nix/nix-2.29.1/install
-          extra_nix_config: |
-            substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
-            trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+        uses: ./.github/actions/nix-install-ephemeral
 
       - name: Run checks if triggered manually
         if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -76,47 +72,25 @@ jobs:
             exit 1
           fi
 
-      - name: Set PostgreSQL version environment variable
-        run: |
-          echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV
-          echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV
-
-      - name: Generate common-nix.vars.pkr.hcl
-        run: |
-          PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)
-          PG_VERSION=$(echo "$PG_VERSION" | tr -d '"')  # Remove any surrounding quotes
-          echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
-          # Ensure there's a newline at the end of the file
-          echo "" >> common-nix.vars.pkr.hcl
-
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
-
-      - name: Build AMI stage 2
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl
-          POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' stage2-nix-psql.pkr.hcl
+      - name: Build AMI
+        id: build-ami
+        uses: ./.github/actions/build-ami
+        with:
+          postgres_version: ${{ matrix.postgres_version }}
+          region: us-east-1
+          ami_regions: '["us-east-1"]'
+          git_sha: ${{ github.sha }}
 
       - name: Grab release version
         id: process_release_version
         run: |
-          VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g')
-          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          VERSION="${{ steps.build-ami.outputs.postgres_release_version }}"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
 
       - name: Create nix flake revision tarball
         run: |
           GIT_SHA=${{github.sha}}
-          MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }}
+          MAJOR_VERSION=${{ matrix.postgres_version }}
 
           mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}"
           echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version"
@@ -134,7 +108,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ matrix.postgres_version }}" \
             manifest-playbook.yml
 
       - name: Upload nix flake revision to s3 staging
@@ -155,7 +129,7 @@ jobs:
           ansible-playbook -i localhost \
             -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \
             -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \
-            -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \
+            -e "postgres_major_version=${{ matrix.postgres_version }}" \
             manifest-playbook.yml
     
       - name: Upload nix flake revision to s3 prod
@@ -184,9 +158,11 @@ jobs:
       - name: Cleanup resources after build
         if: ${{ always() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
 
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
+          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
           aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml
diff --git a/nix/hooks.nix b/nix/hooks.nix
