Skip to content

Commit 152c7f2

Browse files
agneumagneum
committed
chore: refactor synthetic tests configuration:
* install yq * replace sed instructions to yq rules to make configuration more reliable
1 parent f9df799 commit 152c7f2

File tree

6 files changed

+116
-79
lines changed

6 files changed

+116
-79
lines changed

‎test/1.synthetic.sh‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ set -euxo pipefail
33

44
TAG=${TAG:-${CI_COMMIT_REF_SLUG}}
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6-
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
6+
7+
# Environment variables for replacement rules
8+
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
79

810
DIR=${0%/*}
911

@@ -58,17 +60,18 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-mast
5860
--output "${configDir}/server.yml"
5961

6062
# Edit the following options
61-
sed -ri 's/^(\s*)(debug:.*$)/\1debug: true/' "${configDir}/server.yml"
62-
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
63-
sed -ri 's/^(\s*)(- logicalDump$)/\1/' "${configDir}/server.yml"
64-
sed -ri 's/^(\s*)(- logicalRestore$)/\1/' "${configDir}/server.yml"
65-
# replace postgres version
66-
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
67-
63+
yq eval -i '
64+
.global.debug = true |
65+
.global.telemetry.enabled = false |
66+
.localUI.enabled = false |
67+
del(.retrieval.jobs[] | select(. == "logicalDump")) |
68+
del(.retrieval.jobs[] | select(. == "logicalRestore")) |
69+
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
70+
' "${configDir}/server.yml"
6871

6972
# logerrors is not supported in PostgreSQL 9.6
7073
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
71-
sed -ri 's/^(\s*)(shared_preload_libraries:.*$)/\1shared_preload_libraries: "pg_stat_statements, auto_explain"/' "${configDir}/server.yml"
74+
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
7275
fi
7376

7477
## Launch Database Lab server

‎test/2.logical_generic.sh‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ set -euxo pipefail
33

44
TAG="${TAG:-"master"}"
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6-
SOURCE_DBNAME="${SOURCE_DBNAME:-test}"
7-
SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
8-
SOURCE_PORT="${SOURCE_PORT:-7432}"
9-
SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
10-
SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
11-
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
6+
7+
# Environment variables for replacement rules
8+
export SOURCE_DBNAME="${SOURCE_DBNAME:-test}"
9+
export SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
10+
export SOURCE_PORT="${SOURCE_PORT:-7432}"
11+
export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
12+
export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
13+
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
1214

1315
DIR=${0%/*}
1416

@@ -55,19 +57,21 @@ metaDir="$HOME/.dblab/engine/meta"
5557
# Copy the contents of configuration example
5658
mkdir -p "${configDir}"
5759

58-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_generic.yml \
60+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.logical_generic.yml \
5961
--output "${configDir}/server.yml"
6062

6163
# Edit the following options
62-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
63-
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
64-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
65-
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${SOURCE_HOST}/" "${configDir}/server.yml"
66-
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${SOURCE_PORT}/" "${configDir}/server.yml"
67-
sed -ri "s/^(\s*)(username: postgres$)/\1username: ${SOURCE_USERNAME}/" "${configDir}/server.yml"
68-
sed -ri "s/^(\s*)(password:.*$)/\1password: ${SOURCE_PASSWORD}/" "${configDir}/server.yml"
69-
# replace postgres version
70-
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
64+
yq eval -i '
65+
.global.debug = true |
66+
.global.telemetry.enabled = false |
67+
.localUI.enabled = false |
68+
.retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) |
69+
.retrieval.spec.logicalDump.options.source.connection.host = strenv(SOURCE_HOST) |
70+
.retrieval.spec.logicalDump.options.source.connection.port = env(SOURCE_PORT) |
71+
.retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) |
72+
.retrieval.spec.logicalDump.options.source.connection.password = strenv(SOURCE_PASSWORD) |
73+
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
74+
' "${configDir}/server.yml"
7175

7276
## Launch Database Lab server
7377
sudo docker run \

‎test/3.physical_walg.sh‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,18 @@ set -euxo pipefail
33

44
TAG="${TAG:-"master"}"
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6-
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
76

8-
WALG_BACKUP_NAME="${WALG_BACKUP_NAME:-"LATEST"}"
9-
# AWS
7+
# Environment variables for replacement rules
8+
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
9+
export WALG_BACKUP_NAME="${WALG_BACKUP_NAME:-"LATEST"}"
10+
## AWS
1011
set +euxo pipefail # ---- do not display secrets
11-
AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-""}"
12-
AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}"
13-
WALG_S3_PREFIX="${WALG_S3_PREFIX:-""}"
14-
# GS
15-
WALG_GS_PREFIX="${WALG_GS_PREFIX:-""}"
16-
GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-""}"
12+
export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-""}"
13+
export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}"
14+
export WALG_S3_PREFIX="${WALG_S3_PREFIX:-""}"
15+
## GS
16+
export WALG_GS_PREFIX="${WALG_GS_PREFIX:-""}"
17+
export GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-""}"
1718
# check variables
1819
[ -z "${WALG_S3_PREFIX}" ] && [ -z "${WALG_GS_PREFIX}" ] && echo "Variables not specified" && exit 1
1920
set -euxo pipefail # ----
@@ -33,29 +34,37 @@ metaDir="$HOME/.dblab/engine/meta"
3334
# Copy the contents of configuration example
3435
mkdir -p "${configDir}"
3536

36-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.physical_walg.yml \
37+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.physical_walg.yml \
3738
--output "${configDir}/server.yml"
3839

3940
# Edit the following options
40-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
41-
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
42-
# set WAL-G envs
43-
sed -ri "s/^(\s*)(backupName:.*$)/\1backupName: ${WALG_BACKUP_NAME}/" "${configDir}/server.yml"
41+
yq eval -i '
42+
.global.debug = true |
43+
.global.telemetry.enabled = false |
44+
.localUI.enabled = false |
45+
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
46+
.retrieval.spec.physicalRestore.options.walg.backupName = strenv(WALG_BACKUP_NAME) |
47+
.retrieval.spec.physicalRestore.options.sync.configs.shared_buffers = "512MB" |
48+
.retrieval.spec.physicalSnapshot.options.skipStartSnapshot = true
49+
' "${configDir}/server.yml"
50+
4451
set +euxo pipefail # ---- do not display secrets
4552
if [ -n "${WALG_S3_PREFIX}" ] ; then
46-
sed -ri "s/^(\s*)(WALG_GS_PREFIX:.*$)/\1AWS_ACCESS_KEY_ID: \"${AWS_ACCESS_KEY_ID}\" \n AWS_SECRET_ACCESS_KEY: \"${AWS_SECRET_ACCESS_KEY}\"\n WALG_S3_PREFIX: \"${WALG_S3_PREFIX}\"/" "${configDir}/server.yml"
47-
sed -i "/GOOGLE_APPLICATION_CREDENTIALS/d" "${configDir}/server.yml"
53+
yq eval -i '
54+
del(.retrieval.spec.physicalRestore.options.envs.WALG_GS_PREFIX) |
55+
del(.retrieval.spec.physicalRestore.options.envs.GOOGLE_APPLICATION_CREDENTIALS) |
56+
.retrieval.spec.physicalRestore.options.envs.AWS_ACCESS_KEY_ID = strenv(AWS_ACCESS_KEY_ID) |
57+
.retrieval.spec.physicalRestore.options.envs.AWS_SECRET_ACCESS_KEY = strenv(AWS_SECRET_ACCESS_KEY) |
58+
.retrieval.spec.physicalRestore.options.envs.WALG_S3_PREFIX = strenv(WALG_S3_PREFIX)
59+
' "${configDir}/server.yml"
60+
4861
elif [ -n "${WALG_GS_PREFIX}" ] ; then
49-
sed -ri "s/^(\s*)(WALG_GS_PREFIX:.*$)/\1WALG_GS_PREFIX: \"${WALG_GS_PREFIX}\"/" "${configDir}/server.yml"
50-
sed -ri "s/^(\s*)(GOOGLE_APPLICATION_CREDENTIALS:.*$)/\1GOOGLE_APPLICATION_CREDENTIALS: \"${GOOGLE_APPLICATION_CREDENTIALS}\"/" "${configDir}/server.yml"
62+
yq eval -i '
63+
.retrieval.spec.physicalRestore.options.envs.WALG_GS_PREFIX = strenv(WALG_GS_PREFIX) |
64+
.retrieval.spec.physicalRestore.options.envs.GOOGLE_APPLICATION_CREDENTIALS = strenv(GOOGLE_APPLICATION_CREDENTIALS)
65+
' "${configDir}/server.yml"
5166
fi
5267
set -euxo pipefail # ----
53-
# replace postgres version
54-
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
55-
# reduce shared_buffers (optional)
56-
sed -ri "s/^(\s*)(shared_buffers:.*$)/\1shared_buffers: 512MB/" "${configDir}/server.yml"
57-
# skip snapshotting on start to replace some Postgres parameters after PGDATA receiving
58-
sed -ri "s/^(\s*)(skipStartSnapshot:.*$)/\1skipStartSnapshot: true/" "${configDir}/server.yml"
5968

6069
## Launch Database Lab server
6170
sudo docker run \

‎test/4.physical_basebackup.sh‎

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ set -euxo pipefail
33

44
TAG="${TAG:-"master"}"
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6-
SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
7-
SOURCE_PORT="${SOURCE_PORT:-7432}"
8-
SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
9-
SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
10-
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
6+
7+
# Environment variables for replacement rules
8+
export SOURCE_HOST="${SOURCE_HOST:-172.17.0.1}"
9+
export SOURCE_PORT="${SOURCE_PORT:-7432}"
10+
export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
11+
export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
12+
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
1113

1214
DIR=${0%/*}
1315

@@ -59,18 +61,24 @@ metaDir="$HOME/.dblab/engine/meta"
5961
# Copy the contents of configuration example
6062
mkdir -p "${configDir}"
6163

62-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.physical_generic.yml \
64+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.physical_generic.yml \
6365
--output "${configDir}/server.yml"
6466

6567
# Edit the following options
66-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
67-
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
68-
sed -ri "s/^(\s*)(PGUSER:.*$)/\1PGUSER: ${SOURCE_USERNAME}/" "${configDir}/server.yml"
69-
sed -ri "s/^(\s*)(PGPASSWORD:.*$)/\1PGPASSWORD: ${SOURCE_PASSWORD}/" "${configDir}/server.yml"
70-
sed -ri "s/^(\s*)(PGHOST:.*$)/\1PGHOST: ${SOURCE_HOST}/" "${configDir}/server.yml"
71-
sed -ri "s/^(\s*)(PGPORT:.*$)/\1PGPORT: ${SOURCE_PORT}/" "${configDir}/server.yml"
72-
# replace postgres version
73-
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
68+
yq eval -i '
69+
.global.debug = true |
70+
.global.telemetry.enabled = false |
71+
.localUI.enabled = false |
72+
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
73+
.retrieval.spec.physicalRestore.options.envs.PGUSER = strenv(SOURCE_USERNAME) |
74+
.retrieval.spec.physicalRestore.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) |
75+
.retrieval.spec.physicalRestore.options.envs.PGHOST = strenv(SOURCE_HOST) |
76+
.retrieval.spec.physicalRestore.options.envs.PGPORT = env(SOURCE_PORT) |
77+
.retrieval.spec.physicalSnapshot.options.envs.PGUSER = strenv(SOURCE_USERNAME) |
78+
.retrieval.spec.physicalSnapshot.options.envs.PGPASSWORD = strenv(SOURCE_PASSWORD) |
79+
.retrieval.spec.physicalSnapshot.options.envs.PGHOST = strenv(SOURCE_HOST) |
80+
.retrieval.spec.physicalSnapshot.options.envs.PGPORT = env(SOURCE_PORT)
81+
' "${configDir}/server.yml"
7482

7583
## Launch Database Lab server
7684
sudo docker run \

‎test/5.logical_rds.sh‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,16 @@ set -euxo pipefail
33

44
TAG="${TAG:-"master"}"
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6-
POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
7-
SOURCE_DBNAME="${SOURCE_DBNAME:-"test"}"
8-
SOURCE_USERNAME="${SOURCE_USERNAME:-"test_user"}"
9-
AWS_REGION="${AWS_REGION:-"us-east-2"}"
10-
RDS_DB_IDENTIFIER="${RDS_DB_IDENTIFIER:-"logical-rds-test1"}"
6+
7+
# Environment variables for replacement rules
8+
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
9+
export SOURCE_DBNAME="${SOURCE_DBNAME:-"test"}"
10+
export SOURCE_USERNAME="${SOURCE_USERNAME:-"test_user"}"
11+
export AWS_REGION="${AWS_REGION:-"us-east-2"}"
12+
export RDS_DB_IDENTIFIER="${RDS_DB_IDENTIFIER:-"logical-rds-test1"}"
1113
set +euxo pipefail # ---- do not display secrets
12-
AWS_ACCESS_KEY="${AWS_ACCESS_KEY:-""}"
13-
AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}"
14+
export AWS_ACCESS_KEY="${AWS_ACCESS_KEY:-""}"
15+
export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-""}"
1416
set -euxo pipefail # ----
1517

1618
DIR=${0%/*}
@@ -28,18 +30,20 @@ metaDir="$HOME/.dblab/engine/meta"
2830
# Copy the contents of configuration example
2931
mkdir -p "${configDir}"
3032

31-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${TAG}"/configs/config.example.logical_rds_iam.yml \
33+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/"${CI_COMMIT_BRANCH:-master}"/configs/config.example.logical_rds_iam.yml \
3234
--output "${configDir}/server.yml"
3335

3436
# Edit the following options
35-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: true/" "${configDir}/server.yml"
36-
sed -ri '/^ *telemetry:/,/^ *[^:]*:/s/enabled: true/enabled: false/' "${configDir}/server.yml"
37-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${SOURCE_DBNAME}/" "${configDir}/server.yml"
38-
sed -ri "s/^(\s*)(username: test_user.*$)/\1username: \"${SOURCE_USERNAME}\"/" "${configDir}/server.yml"
39-
sed -ri "s/^(\s*)(awsRegion:.*$)/\1awsRegion: \"${AWS_REGION}\"/" "${configDir}/server.yml"
40-
sed -ri "s/^(\s*)(dbInstanceIdentifier:.*$)/\1dbInstanceIdentifier: \"${RDS_DB_IDENTIFIER}\"/" "${configDir}/server.yml"
41-
# replace postgres version
42-
sed -ri "s/:13/:${POSTGRES_VERSION}/g" "${configDir}/server.yml"
37+
yq eval -i '
38+
.global.debug = true |
39+
.global.telemetry.enabled = false |
40+
.localUI.enabled = false |
41+
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION) |
42+
.retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) |
43+
.retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) |
44+
.retrieval.spec.logicalDump.options.source.rdsIam.awsRegion = strenv(AWS_REGION) |
45+
.retrieval.spec.logicalDump.options.source.rdsIam.dbInstanceIdentifier = strenv(RDS_DB_IDENTIFIER)
46+
' "${configDir}/server.yml"
4347

4448
# Download AWS RDS certificate
4549
curl https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \

‎test/_prerequisites.ubuntu.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,12 @@ sudo apt-get install -y \
2929
# Install psql
3030
sudo apt-get install -y \
3131
postgresql-client
32+
33+
# Install yq
34+
if ! command -v yq &> /dev/null; then
35+
sudo curl -L https://github.com/mikefarah/yq/releases/download/v4.14.2/yq_linux_amd64 -o /usr/bin/yq && sudo chmod +x /usr/bin/yq
36+
fi
37+
# TODO: Uncomment after upgrade the runner machine
38+
# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
39+
# sudo add-apt-repository ppa:rmescandon/yq
40+
# sudo apt-get update && sudo apt-get install yq -y

0 commit comments

Comments
 (0)