diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index bb6641ff0..c6154b539 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -61,14 +61,6 @@ jobs: # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - name: Build AMI stage 1 - env: - POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} - run: | - GIT_SHA=${{ steps.get_sha.outputs.sha }} - nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - - name: Build AMI stage 2 env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 3654de90f..86cbba894 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -89,15 +89,6 @@ jobs: # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - name: Build AMI stage 1 - env: - POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} - run: | - GIT_SHA=${{github.sha}} - nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl - # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined? - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' amazon-arm64-nix.pkr.hcl - - name: Build AMI stage 2 env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 2677d1ce0..519e96dde 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -108,15 +108,6 @@ jobs: # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - name: Build AMI stage 1 - env: - AWS_MAX_ATTEMPTS: 10 - AWS_RETRY_MODE: adaptive - run: | - GIT_SHA=${{github.sha}} - nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - - name: Build AMI stage 2 env: AWS_MAX_ATTEMPTS: 10 diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 789a48538..e4bc9e911 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -92,6 +92,18 @@ variable "force-deregister" { default = false } +variable "base-image-nightly" { + type = bool + default = false + description = "Build as version-agnostic base image for nightly" +} + +variable "build-timestamp" { + type = string + default = "" + description = "Timestamp for nightly builds" +} + packer { required_plugins { amazon = { @@ -106,7 +118,7 @@ source "amazon-ebssurrogate" "source" { profile = "${var.profile}" #access_key = "${var.aws_access_key}" #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" - ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" + ami_name = var.base-image-nightly ? "${var.ami_name}-base-stage-1-${var.build-timestamp}-nightly" : "${var.ami_name}-${var.postgres-version}-stage-1" ami_virtualization_type = "hvm" ami_architecture = "arm64" ami_regions = "${var.ami_regions}" @@ -170,8 +182,10 @@ source "amazon-ebssurrogate" "source" { tags = { creator = "packer" appType = "postgres" - postgresVersion = "${var.postgres-version}-stage1" + postgresVersion = var.base-image-nightly ? "base-nightly" : "${var.postgres-version}-stage1" sourceSha = "${var.git-head-version}" + buildTimestamp = var.base-image-nightly ? "${var.build-timestamp}" : "" + buildType = var.base-image-nightly ? "nightly" : "release" } communicator = "ssh" diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 5effa0fdc..9b4e7ce80 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -34,7 +34,7 @@ tags: - install-pgbouncer - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml @@ -45,44 +45,44 @@ tags: - install-gotrue - install-supabase-internal - when: debpkg_mode or nixpkg_mode - + when: debpkg_mode or nixpkg_mode or stage2_nix + - name: Install PostgREST import_tasks: tasks/setup-postgrest.yml tags: - install-postgrest - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or stage2_nix - name: Install Envoy import_tasks: tasks/setup-envoy.yml tags: - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Install Kong import_tasks: tasks/setup-kong.yml tags: - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Install nginx import_tasks: tasks/setup-nginx.yml tags: - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Install Supabase specific content import_tasks: tasks/setup-supabase-internal.yml tags: - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Fix IPv6 NDisc issues (disabled) import_tasks: tasks/fix-ipv6-ndisc.yml tags: - install-supabase-internal - when: (debpkg_mode or nixpkg_mode) and (qemu_mode is undefined) + when: (debpkg_mode or nixpkg_mode or stage2_nix) and (qemu_mode is undefined) - name: Start Postgres Database without Systemd become: yes @@ -95,7 +95,7 @@ copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Transfer init SQL files copy: @@ -136,11 +136,11 @@ import_tasks: tasks/finalize-ami.yml tags: - install-supabase-internal - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Enhance fail2ban import_tasks: tasks/setup-fail2ban.yml - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or nixpkg_mode or stage2_nix # Install EC2 instance connect diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index a796f98d1..79996dd43 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -135,3 +135,11 @@ - name: PgBouncer - reload systemd ansible.builtin.systemd_service: daemon_reload: true + +- name: PgBouncer - create log file + ansible.builtin.file: + path: '/var/log/pgbouncer.log' + state: 'touch' + owner: 'pgbouncer' + group: 'postgres' + mode: '0644' diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 16e83eea2..f3dad9844 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -290,9 +290,19 @@ owner: 'postgres' path: '/run/postgresql' state: 'directory' + when: + - stage2_nix or (stage2_nix and qemu_mode is defined) + +- name: Create postgresql log directory for stage2_nix + become: true + ansible.builtin.file: + group: 'postgres' + owner: 'postgres' + path: '/var/log/postgresql' + state: 'directory' + mode: '0750' when: - stage2_nix - - qemu_mode is defined - name: Restart Postgres Database without Systemd become: true diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 7f8048f90..0b623ed27 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -83,3 +83,27 @@ regexp: "#include = '/etc/postgresql-custom/wal-g.conf'" replace: "include = '/etc/postgresql-custom/wal-g.conf'" become: true + + - name: Create wal-g log directory + ansible.builtin.file: + path: '/var/log/wal-g' + state: 'directory' + owner: 'postgres' + group: 'postgres' + mode: '0300' + + - name: Create wal-g log files + ansible.builtin.file: + path: "/var/log/wal-g/{{ walg_log_item }}" + state: 'touch' + owner: 'postgres' + group: 'postgres' + mode: '0300' + loop: + - 'backup-push.log' + - 'backup-fetch.log' + - 'wal-push.log' + - 'wal-fetch.log' + - 'pitr.log' + loop_control: + loop_var: 'walg_log_item' diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index ea6e157b6..3f6b49328 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -16,9 +16,10 @@ become: true become_user: 'postgres' loop: - - { in: "^(shared_preload_libraries = '.*)pgsodium(.*')", out: '\1\2' } - - { in: "^(shared_preload_libraries = '.*)supabase_vault(.*')", out: '\1\2' } - - { in: "^(shared_preload_libraries = '.*)*supabase_vault(.*')", out: '\1\2' } + - { in: "^(shared_preload_libraries = '.*),\\s*pgsodium(.*')", out: '\1\2' } + - { in: "^(shared_preload_libraries = '.*)pgsodium,\\s*(.*')", out: '\1\2' } + - { in: "^(shared_preload_libraries = '.*),\\s*supabase_vault(.*')", out: '\1\2' } + - { in: "^(shared_preload_libraries = '.*)supabase_vault,\\s*(.*')", out: '\1\2' } - { in: '^(pgsodium\.getkey_script=)', out: '#\1' } loop_control: loop_var: 'regx' diff --git a/ansible/vars.yml b/ansible/vars.yml index 84035bca5..db31045d3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,9 +10,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.6.0.014-orioledb" - postgres17: "17.6.1.057" - postgres15: "15.14.1.057" + postgresorioledb-17: "17.6.0.014-orioledb-nb-1" + postgres17: "17.6.1.057-nb-1" + postgres15: "15.14.1.057-nb-1" # Non Postgres Extensions pgbouncer_release: 1.19.0 diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index f478e5b42..aceb46ec0 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -303,11 +303,10 @@ tee /etc/ansible/ansible.cfg <