feat: move stage 1 to nightly build

.github/workflows/ami-release-nix-single.yml

@@ -61,14 +61,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{ steps.get_sha.outputs.sha }}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}"  amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}

.github/workflows/ami-release-nix.yml

@@ -89,15 +89,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined?
-          nix  run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}"  -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }}

.github/workflows/testinfra-ami-build.yml

@@ -108,15 +108,6 @@ jobs:
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
 
-      - name: Build AMI stage 1
-        env:
-          AWS_MAX_ATTEMPTS: 10
-          AWS_RETRY_MODE: adaptive
-        run: |
-          GIT_SHA=${{github.sha}}
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
-          nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
-
       - name: Build AMI stage 2
         env:
           AWS_MAX_ATTEMPTS: 10

amazon-arm64-nix.pkr.hcl

@@ -92,6 +92,18 @@ variable "force-deregister" {
   default = false
 }
 
+variable "base-image-nightly" {
+  type    = bool
+  default = false
+  description = "Build as version-agnostic base image for nightly"
+}
+
+variable "build-timestamp" {
+  type    = string
+  default = ""
+  description = "Timestamp for nightly builds"
+}
+
 packer {
   required_plugins {
     amazon = {
@@ -106,7 +118,7 @@ source "amazon-ebssurrogate" "source" {
   profile = "${var.profile}"
   #access_key    = "${var.aws_access_key}"
   #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}"
-  ami_name = "${var.ami_name}-${var.postgres-version}-stage-1"
+  ami_name = var.base-image-nightly ? "${var.ami_name}-base-stage-1-${var.build-timestamp}-nightly" : "${var.ami_name}-${var.postgres-version}-stage-1"
   ami_virtualization_type = "hvm"
   ami_architecture = "arm64"
   ami_regions   = "${var.ami_regions}"
@@ -170,8 +182,10 @@ source "amazon-ebssurrogate" "source" {
   tags = {
     creator = "packer"
     appType = "postgres"
-    postgresVersion = "${var.postgres-version}-stage1"
+    postgresVersion = var.base-image-nightly ? "base-nightly" : "${var.postgres-version}-stage1"
     sourceSha = "${var.git-head-version}"
+    buildTimestamp = var.base-image-nightly ? "${var.build-timestamp}" : ""
+    buildType = var.base-image-nightly ? "nightly" : "release"
   }
 
   communicator = "ssh"

ansible/playbook.yml

@@ -34,7 +34,7 @@
       tags:
         - install-pgbouncer
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install WAL-G
       import_tasks: tasks/setup-wal-g.yml
@@ -45,44 +45,44 @@
       tags:
         - install-gotrue
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
-  
+      when: debpkg_mode or nixpkg_mode or stage2_nix
+
     - name: Install PostgREST
       import_tasks: tasks/setup-postgrest.yml
       tags:
         - install-postgrest
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or stage2_nix
 
     - name: Install Envoy
       import_tasks: tasks/setup-envoy.yml
       tags:
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install Kong
       import_tasks: tasks/setup-kong.yml
       tags:
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install nginx
       import_tasks: tasks/setup-nginx.yml
       tags:
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install Supabase specific content
       import_tasks: tasks/setup-supabase-internal.yml
       tags:
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Fix IPv6 NDisc issues (disabled)
       import_tasks: tasks/fix-ipv6-ndisc.yml
       tags:
         - install-supabase-internal
-      when: (debpkg_mode or nixpkg_mode) and (qemu_mode is undefined)
+      when: (debpkg_mode or nixpkg_mode or stage2_nix) and (qemu_mode is undefined)
 
     - name: Start Postgres Database without Systemd
       become: yes
@@ -95,7 +95,7 @@
       copy:
         src: files/apt_periodic
         dest: /etc/apt/apt.conf.d/10periodic
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Transfer init SQL files
       copy:
@@ -136,11 +136,11 @@
       import_tasks: tasks/finalize-ami.yml
       tags:
         - install-supabase-internal
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Enhance fail2ban
       import_tasks: tasks/setup-fail2ban.yml
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
 
     # Install EC2 instance connect

ansible/tasks/setup-pgbouncer.yml

@@ -135,3 +135,11 @@
 - name: PgBouncer - reload systemd
   ansible.builtin.systemd_service:
     daemon_reload: true
+
+- name: PgBouncer - create log file
+  ansible.builtin.file:
+    path: '/var/log/pgbouncer.log'
+    state: 'touch'
+    owner: 'pgbouncer'
+    group: 'postgres'
+    mode: '0644'

ansible/tasks/setup-postgres.yml

@@ -290,9 +290,19 @@
     owner: 'postgres'
     path: '/run/postgresql'
     state: 'directory'
+  when:
+    - stage2_nix or (stage2_nix and qemu_mode is defined)
+
+- name: Create postgresql log directory for stage2_nix
+  become: true
+  ansible.builtin.file:
+    group: 'postgres'
+    owner: 'postgres'
+    path: '/var/log/postgresql'
+    state: 'directory'
+    mode: '0750'
   when:
     - stage2_nix
-    - qemu_mode is defined
 
 - name: Restart Postgres Database without Systemd
   become: true

ansible/tasks/setup-wal-g.yml

@@ -83,3 +83,27 @@
         regexp: "#include = '/etc/postgresql-custom/wal-g.conf'"
         replace: "include = '/etc/postgresql-custom/wal-g.conf'"
       become: true
+
+    - name: Create wal-g log directory
+      ansible.builtin.file:
+        path: '/var/log/wal-g'
+        state: 'directory'
+        owner: 'postgres'
+        group: 'postgres'
+        mode: '0300'
+
+    - name: Create wal-g log files
+      ansible.builtin.file:
+        path: "/var/log/wal-g/{{ walg_log_item }}"
+        state: 'touch'
+        owner: 'postgres'
+        group: 'postgres'
+        mode: '0300'
+      loop:
+        - 'backup-push.log'
+        - 'backup-fetch.log'
+        - 'wal-push.log'
+        - 'wal-fetch.log'
+        - 'pitr.log'
+      loop_control:
+        loop_var: 'walg_log_item'

ansible/tasks/test-image.yml

@@ -16,9 +16,10 @@
       become: true
       become_user: 'postgres'
       loop:
-        - { in: "^(shared_preload_libraries = '.*)pgsodium(.*')", out: '\1\2' }
-        - { in: "^(shared_preload_libraries = '.*)supabase_vault(.*')", out: '\1\2' }
-        - { in: "^(shared_preload_libraries = '.*)*supabase_vault(.*')", out: '\1\2' }
+        - { in: "^(shared_preload_libraries = '.*),\\s*pgsodium(.*')", out: '\1\2' }
+        - { in: "^(shared_preload_libraries = '.*)pgsodium,\\s*(.*')", out: '\1\2' }
+        - { in: "^(shared_preload_libraries = '.*),\\s*supabase_vault(.*')", out: '\1\2' }
+        - { in: "^(shared_preload_libraries = '.*)supabase_vault,\\s*(.*')", out: '\1\2' }
         - { in: '^(pgsodium\.getkey_script=)', out: '#\1' }
       loop_control:
         loop_var: 'regx'

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.014-orioledb"
-  postgres17: "17.6.1.057"
-  postgres15: "15.14.1.057"
+  postgresorioledb-17: "17.6.0.014-orioledb-nb-1"
+  postgres17: "17.6.1.057-nb-1"
+  postgres15: "15.14.1.057-nb-1"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.19.0

ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -303,11 +303,10 @@ tee /etc/ansible/ansible.cfg <<EOF
 callbacks_enabled = timer, profile_tasks, profile_roles
 EOF
 	# Run Ansible playbook
-	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp 
+	#export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp
 	ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \
 	--extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \
-	--extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \
 	$ARGS
 }
 
@@ -338,29 +337,9 @@ function clean_system {
 	# https://github.com/fail2ban/fail2ban/issues/1593
 	touch /mnt/var/log/auth.log
 
-	touch /mnt/var/log/pgbouncer.log
-	if [ -f /usr/bin/chown ]; then
-		chroot /mnt /usr/bin/chown pgbouncer:postgres /var/log/pgbouncer.log
-	fi
-
-	# Setup postgresql logs
-	mkdir -p /mnt/var/log/postgresql
-	if [ -f /usr/bin/chown ]; then
-		chroot /mnt /usr/bin/chown postgres:postgres /var/log/postgresql
-	fi
-
-	# Setup wal-g logs
-	mkdir /mnt/var/log/wal-g
-	touch /mnt/var/log/wal-g/{backup-push.log,backup-fetch.log,wal-push.log,wal-fetch.log,pitr.log}
-
-	#Creatre Sysstat directory for SAR
+	# Create Sysstat directory for SAR
 	mkdir /mnt/var/log/sysstat
 
-	if [ -f /usr/bin/chown ]; then
-		chroot /mnt /usr/bin/chown -R postgres:postgres /var/log/wal-g
-		chroot /mnt /usr/bin/chmod -R 0300 /var/log/wal-g
-	fi
-
 	# audit logs directory for apparmor
 	mkdir /mnt/var/log/audit
 

scripts/nix-provision.sh

@@ -51,6 +51,7 @@ EOF
         --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \
         --extra-vars "nix_secret_key=${NIX_SECRET_KEY}" \
         --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \
+        --extra-vars "postgresql_major=${POSTGRES_MAJOR_VERSION}" \
         $ARGS
 }
 

stage2-nix-psql.pkr.hcl

@@ -66,12 +66,12 @@ source "amazon-ebs" "ubuntu" {
   region        = "${var.region}"
   source_ami_filter {
     filters = {
-      name   = "${var.ami_name}-${var.postgres-version}-stage-1"
+      name   = "${var.ami_name}-base-stage-1-*-nightly"
       root-device-type    = "ebs"
       virtualization-type = "hvm"
     }
     most_recent = true
-    owners      = ["amazon", "self"]
+    owners      = ["self"]
   }
 
   communicator = "ssh"