Skip to content

Commit 843785f

Browse files
hunleydhunleyd
committed
Merge remote-tracking branch 'origin/develop' into PSQL-822
* origin/develop: refactor: Refactor PostgreSQL cron config and deprecations (#1951) chore: bump Wrappers version to 0.5.7 (#1945) chore: bump versions (#1949) Revert "docs: create an automated changlog file that shows up in release file…" (#1948) chore: remove precreated Storage objects (#1911) refactor: Move PostgreSQL auto_explain config to conf.d (#1943) docs: create an automated changlog file that shows up in release files (#1793) fix: removing dead nix code (#1946) docs: expansion to help understand anatomy and code api in user here (#1944) fix: try to use mirror fallbacks for docker images (#1942) feat: need this workflow present to test (#1941) feat: Add PostgreSQL conf.d directory (#1929) [jj-spr] initial version (#1866)
2 parents 3b56c40 + a7327e0 commit 843785f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+2210
-2142
lines changed

‎.github/actions/nix-install-ephemeral/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,4 @@ runs:
4444
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
4545
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
4646
${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }}
47-
max-jobs = 8
47+
max-jobs = 4

‎.github/workflows/base-image-nightly.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Build Base Image Nightly
2+
3+
on:
4+
#schedule:
5+
# - cron: '0 2 * * *' # 2 AM UTC daily
6+
workflow_dispatch:
7+
inputs:
8+
branch:
9+
description: 'Branch to build from'
10+
required: false
11+
default: 'develop'
12+
type: string
13+
14+
permissions:
15+
contents: read
16+
id-token: write
17+
18+
jobs:
19+
build-base-image:
20+
runs-on: blacksmith-4vcpu-ubuntu-2404-arm
21+
timeout-minutes: 150
22+
23+
steps:
24+
- name: Checkout Repo
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
26+
with:
27+
ref: ${{ github.event.inputs.branch || 'develop' }}
28+
29+
- name: Configure AWS credentials
30+
uses: aws-actions/configure-aws-credentials@v4
31+
with:
32+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
33+
aws-region: "us-east-1"
34+
output-credentials: true
35+
role-duration-seconds: 7200
36+
37+
- name: Install nix
38+
uses: cachix/install-nix-action@v27
39+
with:
40+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
41+
extra_nix_config: |
42+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
43+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
44+
45+
- name: Set execution ID and timestamp
46+
run: |
47+
echo "EXECUTION_ID=${{ github.run_id }}-base-nightly" >> $GITHUB_ENV
48+
echo "BUILD_TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
49+
50+
- name: Build base stage 1 AMI
51+
env:
52+
AWS_MAX_ATTEMPTS: 10
53+
AWS_RETRY_MODE: adaptive
54+
run: |
55+
GIT_SHA=${{ github.sha }}
56+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
57+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build \
58+
-var "git-head-version=${GIT_SHA}" \
59+
-var "packer-execution-id=${EXECUTION_ID}" \
60+
-var-file="development-arm.vars.pkr.hcl" \
61+
-var "base-image-nightly=true" \
62+
-var "build-timestamp=${BUILD_TIMESTAMP}" \
63+
-var "region=us-east-1" \
64+
-var 'ami_regions=["us-east-1","ap-southeast-1"]' \
65+
amazon-arm64-nix.pkr.hcl
66+
67+
- name: Slack Notification on Failure
68+
if: ${{ failure() }}
69+
uses: rtCamp/action-slack-notify@v2
70+
env:
71+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
72+
SLACK_USERNAME: 'gha-failures-notifier'
73+
SLACK_COLOR: 'danger'
74+
SLACK_MESSAGE: 'Building base image nightly failed'
75+
SLACK_FOOTER: ''
76+
77+
- name: Cleanup resources after build
78+
if: ${{ always() }}
79+
run: |
80+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
81+
82+
- name: Cleanup resources on build cancellation
83+
if: ${{ cancelled() }}
84+
run: |
85+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids

‎.github/workflows/nix-build.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
fail-fast: false
2525
matrix:
2626
include:
27-
- runner: blacksmith-32vcpu-ubuntu-2404
27+
- runner: blacksmith-32vcpu-ubuntu-2404
2828
arch: amd64
2929
- runner: blacksmith-32vcpu-ubuntu-2404-arm
3030
arch: arm64
@@ -66,11 +66,11 @@ jobs:
6666
sudo rm -rf /tmp/* 2>/dev/null || true
6767
echo "=== AFTER CLEANUP ==="
6868
df -h
69-
-
69+
-
7070
name: Build psql bundle
7171
run: >
72-
nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48"
73-
-- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }}
72+
nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48"
73+
-- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} --copy-to "s3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key"
7474
--flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')"
7575
env:
7676
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}

‎Dockerfile-15‎

Lines changed: 58 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ ARG pg_repack_release=1.4.8
3030
ARG vault_release=0.2.8
3131
ARG groonga_release=12.0.8
3232
ARG pgroonga_release=2.4.0
33-
ARG wrappers_release=0.5.6
33+
ARG wrappers_release=0.5.7
3434
ARG hypopg_release=1.3.1
3535
ARG pgvector_release=0.4.0
3636
ARG pg_tle_release=1.3.2
@@ -40,7 +40,56 @@ ARG wal_g_release=2.0.1
4040

4141
FROM ubuntu:noble as base
4242

43-
RUN apt update -y && apt install -y \
43+
# Create reusable apt mirror fallback function
44+
RUN echo '#!/bin/bash\n\
45+
apt_update_with_fallback() {\n\
46+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
47+
local max_attempts=2\n\
48+
local attempt=1\n\
49+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
50+
\n\
51+
for mirror in $mirrors; do\n\
52+
echo "========================================="\n\
53+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
54+
echo "Attempt ${attempt} of ${max_attempts}"\n\
55+
echo "========================================="\n\
56+
\n\
57+
if [ -f "${sources_file}" ]; then\n\
58+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
59+
fi\n\
60+
\n\
61+
if timeout 300 apt-get update 2>&1; then\n\
62+
echo "========================================="\n\
63+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
64+
echo "========================================="\n\
65+
return 0\n\
66+
else\n\
67+
local exit_code=$?\n\
68+
echo "========================================="\n\
69+
echo "✗ Failed to update using mirror: ${mirror}"\n\
70+
echo "Exit code: ${exit_code}"\n\
71+
echo "========================================="\n\
72+
\n\
73+
apt-get clean\n\
74+
rm -rf /var/lib/apt/lists/*\n\
75+
\n\
76+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
77+
local sleep_time=$((attempt * 5))\n\
78+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
79+
sleep ${sleep_time}\n\
80+
fi\n\
81+
fi\n\
82+
\n\
83+
attempt=$((attempt + 1))\n\
84+
done\n\
85+
\n\
86+
echo "========================================="\n\
87+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
88+
echo "========================================="\n\
89+
return 1\n\
90+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
91+
92+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4493
curl \
4594
gnupg \
4695
lsb-release \
@@ -96,13 +145,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
96145
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
97146

98147

99-
RUN apt-get update && \
148+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
100149
apt-get install -y --no-install-recommends tzdata
101150

102151
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
103152
dpkg-reconfigure --frontend noninteractive tzdata
104153

105-
RUN apt-get update && \
154+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
106155
apt-get install -y --no-install-recommends \
107156
build-essential \
108157
checkinstall \
@@ -131,7 +180,7 @@ WORKDIR /
131180
FROM base as gosu
132181
ARG TARGETARCH
133182
# Install dependencies
134-
RUN apt-get update && apt-get install -y --no-install-recommends \
183+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
135184
gnupg \
136185
ca-certificates \
137186
&& rm -rf /var/lib/apt/lists/*
@@ -160,6 +209,7 @@ COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
160209
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
161210
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
162211
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
212+
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
163213
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
164214
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
165215
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
@@ -174,14 +224,11 @@ RUN sed -i \
174224
-e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
175225
-e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
176226
-e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
177-
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
178-
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
179227
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
180228
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
181-
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
182229
usermod -aG postgres wal-g && \
183-
mkdir -p /etc/postgresql-custom && \
184-
chown postgres:postgres /etc/postgresql-custom
230+
mkdir -p /etc/postgresql-custom/conf.d && \
231+
chown -R postgres:postgres /etc/postgresql-custom
185232

186233
# # Include schema migrations
187234
COPY migrations/db /docker-entrypoint-initdb.d/
@@ -205,7 +252,7 @@ EXPOSE 5432
205252
ENV POSTGRES_HOST=/var/run/postgresql
206253
ENV POSTGRES_USER=supabase_admin
207254
ENV POSTGRES_DB=postgres
208-
RUN apt-get update && apt-get install -y --no-install-recommends \
255+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
209256
locales \
210257
&& rm -rf /var/lib/apt/lists/* && \
211258
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

‎Dockerfile-17‎

Lines changed: 58 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ ARG pg_repack_release=1.4.8
3131
ARG vault_release=0.2.8
3232
ARG groonga_release=12.0.8
3333
ARG pgroonga_release=2.4.0
34-
ARG wrappers_release=0.5.6
34+
ARG wrappers_release=0.5.7
3535
ARG hypopg_release=1.3.1
3636
ARG pgvector_release=0.4.0
3737
ARG pg_tle_release=1.3.2
@@ -41,7 +41,56 @@ ARG wal_g_release=3.0.5
4141

4242
FROM ubuntu:noble as base
4343

44-
RUN apt update -y && apt install -y \
44+
# Create reusable apt mirror fallback function
45+
RUN echo '#!/bin/bash\n\
46+
apt_update_with_fallback() {\n\
47+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
48+
local max_attempts=2\n\
49+
local attempt=1\n\
50+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
51+
\n\
52+
for mirror in $mirrors; do\n\
53+
echo "========================================="\n\
54+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
55+
echo "Attempt ${attempt} of ${max_attempts}"\n\
56+
echo "========================================="\n\
57+
\n\
58+
if [ -f "${sources_file}" ]; then\n\
59+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
60+
fi\n\
61+
\n\
62+
if timeout 300 apt-get update 2>&1; then\n\
63+
echo "========================================="\n\
64+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
65+
echo "========================================="\n\
66+
return 0\n\
67+
else\n\
68+
local exit_code=$?\n\
69+
echo "========================================="\n\
70+
echo "✗ Failed to update using mirror: ${mirror}"\n\
71+
echo "Exit code: ${exit_code}"\n\
72+
echo "========================================="\n\
73+
\n\
74+
apt-get clean\n\
75+
rm -rf /var/lib/apt/lists/*\n\
76+
\n\
77+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
78+
local sleep_time=$((attempt * 5))\n\
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
80+
sleep ${sleep_time}\n\
81+
fi\n\
82+
fi\n\
83+
\n\
84+
attempt=$((attempt + 1))\n\
85+
done\n\
86+
\n\
87+
echo "========================================="\n\
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
89+
echo "========================================="\n\
90+
return 1\n\
91+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
92+
93+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4594
curl \
4695
gnupg \
4796
lsb-release \
@@ -100,13 +149,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
100149
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
101150

102151

103-
RUN apt-get update && \
152+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
104153
apt-get install -y --no-install-recommends tzdata
105154

106155
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
107156
dpkg-reconfigure --frontend noninteractive tzdata
108157

109-
RUN apt-get update && \
158+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
110159
apt-get install -y --no-install-recommends \
111160
build-essential \
112161
checkinstall \
@@ -135,7 +184,7 @@ WORKDIR /
135184
FROM base as gosu
136185
ARG TARGETARCH
137186
# Install dependencies
138-
RUN apt-get update && apt-get install -y --no-install-recommends \
187+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
139188
gnupg \
140189
ca-certificates \
141190
&& rm -rf /var/lib/apt/lists/*
@@ -164,6 +213,7 @@ COPY --from=groonga /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins
164213
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf
165214
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
166215
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
216+
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
167217
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
168218
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
169219
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
@@ -178,14 +228,11 @@ RUN sed -i \
178228
-e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
179229
-e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
180230
-e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
181-
echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
182-
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
183231
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
184232
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
185-
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
186233
usermod -aG postgres wal-g && \
187-
mkdir -p /etc/postgresql-custom && \
188-
chown postgres:postgres /etc/postgresql-custom
234+
mkdir -p /etc/postgresql-custom/conf.d && \
235+
chown -R postgres:postgres /etc/postgresql-custom
189236

190237
# Remove items from postgresql.conf
191238
RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
@@ -218,7 +265,7 @@ ENV POSTGRES_HOST=/var/run/postgresql
218265
ENV POSTGRES_USER=supabase_admin
219266
ENV POSTGRES_DB=postgres
220267
ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
221-
RUN apt-get update && apt-get install -y --no-install-recommends \
268+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
222269
locales \
223270
&& rm -rf /var/lib/apt/lists/* && \
224271
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

0 commit comments

Comments
 (0)