Skip to content

Commit 8c0e665

Browse files
agneumagneum
committed
* make integration tests harmless in dev and staging environments:
 * synthetic * logical generic * physical WAL-G * physical basebackup * logical RDS * enable logical generic and physical basebackup cases * install yq from the official repository
1 parent 13716b7 commit 8c0e665

File tree

9 files changed

+259
-106
lines changed

9 files changed

+259
-106
lines changed

‎.gitlab-ci.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,8 +326,9 @@ build-image-swagger-latest:
326326
variables:
327327
IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
328328
script:
329-
- zfs list
330329
- bash test/1.synthetic.sh
330+
- bash test/2.logical_generic.sh
331+
- bash test/4.physical_basebackup.sh
331332
after_script:
332333
- bash test/_cleanup.sh
333334
tags:

‎test/1.synthetic.sh‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
TAG=${TAG:-${CI_COMMIT_REF_SLUG}}
4+
TAG=${TAG:-${CI_COMMIT_REF_SLUG:-"master"}}
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6+
DLE_SERVER_NAME="dblab_server_test"
67

78
# Environment variables for replacement rules
89
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
10+
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
11+
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
12+
export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
13+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
14+
export DLE_TEST_POOL_NAME="test_dblab_pool"
915

1016
DIR=${0%/*}
1117

@@ -17,12 +23,15 @@ source "${DIR}/_zfs.file.sh"
1723
### Step 2. Configure and launch the Database Lab Engine
1824

1925
## Prepare database data directory.
26+
sudo docker rm dblab_pg_initdb || true
27+
2028
sudo docker run \
2129
--name dblab_pg_initdb \
2230
--label dblab_sync \
31+
--label dblab_test \
2332
--env PGDATA=/var/lib/postgresql/pgdata \
2433
--env POSTGRES_HOST_AUTH_METHOD=trust \
25-
--volume /var/lib/dblab/dblab_pool/data:/var/lib/postgresql/pgdata \
34+
--volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/data:/var/lib/postgresql/pgdata \
2635
--detach \
2736
postgres:"${POSTGRES_VERSION}"-alpine
2837

@@ -64,6 +73,10 @@ yq eval -i '
6473
.global.debug = true |
6574
.global.telemetry.enabled = false |
6675
.localUI.enabled = false |
76+
.server.port = env(DLE_SERVER_PORT) |
77+
.provision.portPool.from = env(DLE_PORT_POOL_FROM) |
78+
.provision.portPool.to = env(DLE_PORT_POOL_TO) |
79+
.poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
6780
del(.retrieval.jobs[] | select(. == "logicalDump")) |
6881
del(.retrieval.jobs[] | select(. == "logicalRestore")) |
6982
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
@@ -76,13 +89,14 @@ fi
7689

7790
## Launch Database Lab server
7891
sudo docker run \
79-
--name dblab_server \
92+
--name ${DLE_SERVER_NAME} \
8093
--label dblab_control \
94+
--label dblab_test \
8195
--privileged \
82-
--publish 2345:2345 \
96+
--publish ${DLE_SERVER_PORT}:${DLE_SERVER_PORT} \
8397
--volume /var/run/docker.sock:/var/run/docker.sock \
84-
--volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
85-
--volume /var/lib/dblab:/var/lib/dblab/:rshared \
98+
--volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump:${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump \
99+
--volume ${DLE_TEST_MOUNT_DIR}:${DLE_TEST_MOUNT_DIR}/:rshared \
86100
--volume "${configDir}":/home/dblab/configs:ro \
87101
--volume "${metaDir}":/home/dblab/meta \
88102
--volume /sys/kernel/debug:/sys/kernel/debug:rw \
@@ -93,11 +107,11 @@ sudo docker run \
93107
"${IMAGE2TEST}"
94108

95109
# Check the Database Lab Engine logs
96-
sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
110+
sudo docker logs ${DLE_SERVER_NAME} -f 2>&1 | awk '{print "[CONTAINER ${DLE_SERVER_PORT}]: "$0}' &
97111

98112
### Waiting for the Database Lab Engine initialization.
99113
for i in {1..300}; do
100-
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
114+
curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
101115
sleep 1
102116
done
103117

@@ -113,7 +127,7 @@ dblab --version
113127
# Initialize CLI configuration
114128
dblab init \
115129
--environment-id=test \
116-
--url=http://localhost:2345 \
130+
--url=http://localhost:${DLE_SERVER_PORT} \
117131
--token=secret_token \
118132
--insecure
119133

@@ -129,14 +143,14 @@ dblab clone create \
129143

130144
# Connect to a clone and check the available table
131145
PGPASSWORD=secret_password psql \
132-
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
146+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
133147

134148
# Drop table
135149
PGPASSWORD=secret_password psql \
136-
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
150+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c 'drop table pgbench_accounts'
137151

138152
PGPASSWORD=secret_password psql \
139-
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
153+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
140154

141155
## Reset clone
142156
dblab clone reset testclone
@@ -146,17 +160,17 @@ dblab clone status testclone
146160

147161
# Check the database objects (everything should be the same as when we started)
148162
PGPASSWORD=secret_password psql \
149-
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
163+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
150164

151165

152166
### Step 4. Check clone durability on DLE restart.
153167

154168
## Restart DLE.
155-
sudo docker restart dblab_server
169+
sudo docker restart ${DLE_SERVER_NAME}
156170

157171
### Waiting for the Database Lab Engine to start.
158172
for i in {1..300}; do
159-
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
173+
curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
160174
sleep 1
161175
done
162176

@@ -168,15 +182,15 @@ dblab clone status testclone
168182

169183
# Check the database objects (everything should be the same as when we started)
170184
PGPASSWORD=secret_password psql \
171-
"host=localhost port=6000 user=dblab_user_1 dbname=test" -c '\dt+'
185+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
172186

173187

174188
### Step 5. Destroy clone
175189
dblab clone destroy testclone
176190
dblab clone list
177191

178192
## Stop DLE.
179-
sudo docker stop dblab_server
193+
sudo docker stop ${DLE_SERVER_NAME}
180194

181195
### Finish. clean up
182196
source "${DIR}/_cleanup.sh"

‎test/2.logical_generic.sh‎

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
TAG="${TAG:-"master"}"
4+
TAG=${TAG:-${CI_COMMIT_REF_SLUG:-"master"}}
55
IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
6+
DLE_SERVER_NAME="dblab_server_test"
67

78
# Environment variables for replacement rules
89
export SOURCE_DBNAME="${SOURCE_DBNAME:-test}"
@@ -11,37 +12,52 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
1112
export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
1213
export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
1314
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
15+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
16+
export DLE_TEST_POOL_NAME="test_dblab_pool"
17+
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
18+
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
19+
export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
1420

1521
DIR=${0%/*}
1622

1723
if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
1824
### Step 0. Create source database
19-
sudo rm -rf "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test || true
25+
TMP_DATA_DIR="/tmp/dle_test/logical_generic"
26+
cleanup_testdata_dir() {
27+
sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
28+
}
29+
30+
trap cleanup_testdata_dir EXIT
31+
32+
cleanup_testdata_dir
33+
sudo docker rm postgres"${POSTGRES_VERSION}" || true
34+
2035
sudo docker run \
2136
--name postgres"${POSTGRES_VERSION}" \
2237
--label pgdb \
38+
--label dblab_test \
2339
--privileged \
2440
--publish 172.17.0.1:"${SOURCE_PORT}":5432 \
2541
--env PGDATA=/var/lib/postgresql/pgdata \
2642
--env POSTGRES_USER="${SOURCE_USERNAME}" \
2743
--env POSTGRES_PASSWORD="${SOURCE_PASSWORD}" \
2844
--env POSTGRES_DB="${SOURCE_DBNAME}" \
2945
--env POSTGRES_HOST_AUTH_METHOD=md5 \
30-
--volume "$(pwd)"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
46+
--volume "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test:/var/lib/postgresql/pgdata \
3147
--detach \
3248
postgres:"${POSTGRES_VERSION}-alpine"
3349

3450
for i in {1..300}; do
35-
sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
51+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres -c 'select' > /dev/null 2>&1 && break || echo "test database is not ready yet"
3652
sleep 1
3753
done
3854

3955
# Generate data in the test database using pgbench
4056
# 1,000,000 accounts, ~0.14 GiB of data.
41-
sudo docker exec -it postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
57+
sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
4258

4359
# Database info
44-
sudo docker exec -it postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
60+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
4561
fi
4662

4763
### Step 1. Prepare a machine with disk, Docker, and ZFS
@@ -65,23 +81,35 @@ yq eval -i '
6581
.global.debug = true |
6682
.global.telemetry.enabled = false |
6783
.localUI.enabled = false |
84+
.server.port = env(DLE_SERVER_PORT) |
85+
.poolManager.mountDir = env(DLE_TEST_MOUNT_DIR) |
86+
.provision.portPool.from = env(DLE_PORT_POOL_FROM) |
87+
.provision.portPool.to = env(DLE_PORT_POOL_TO) |
88+
.retrieval.spec.logicalDump.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
6889
.retrieval.spec.logicalDump.options.source.connection.dbname = strenv(SOURCE_DBNAME) |
6990
.retrieval.spec.logicalDump.options.source.connection.host = strenv(SOURCE_HOST) |
7091
.retrieval.spec.logicalDump.options.source.connection.port = env(SOURCE_PORT) |
7192
.retrieval.spec.logicalDump.options.source.connection.username = strenv(SOURCE_USERNAME) |
7293
.retrieval.spec.logicalDump.options.source.connection.password = strenv(SOURCE_PASSWORD) |
94+
.retrieval.spec.logicalRestore.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump" |
7395
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
7496
' "${configDir}/server.yml"
7597

98+
# logerrors is not supported in PostgreSQL 9.6
99+
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
100+
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
101+
fi
102+
76103
## Launch Database Lab server
77104
sudo docker run \
78-
--name dblab_server \
105+
--name ${DLE_SERVER_NAME} \
79106
--label dblab_control \
107+
--label dblab_test \
80108
--privileged \
81-
--publish 2345:2345 \
109+
--publish ${DLE_SERVER_PORT}:${DLE_SERVER_PORT} \
82110
--volume /var/run/docker.sock:/var/run/docker.sock \
83-
--volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
84-
--volume /var/lib/dblab:/var/lib/dblab/:rshared \
111+
--volume ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump:${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/dump \
112+
--volume ${DLE_TEST_MOUNT_DIR}:${DLE_TEST_MOUNT_DIR}/:rshared \
85113
--volume "${configDir}":/home/dblab/configs:ro \
86114
--volume "${metaDir}":/home/dblab/meta \
87115
--volume /sys/kernel/debug:/sys/kernel/debug:rw \
@@ -92,11 +120,11 @@ sudo docker run \
92120
"${IMAGE2TEST}"
93121

94122
# Check the Database Lab Engine logs
95-
sudo docker logs dblab_server -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
123+
sudo docker logs ${DLE_SERVER_NAME} -f 2>&1 | awk '{print "[CONTAINER dblab_server]: "$0}' &
96124

97125
### Waiting for the Database Lab Engine initialization.
98126
for i in {1..300}; do
99-
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
127+
curl http://localhost:${DLE_SERVER_PORT} > /dev/null 2>&1 && break || echo "dblab is not ready yet"
100128
sleep 1
101129
done
102130

@@ -112,7 +140,7 @@ dblab --version
112140
# Initialize CLI configuration
113141
dblab init \
114142
--environment-id=test \
115-
--url=http://localhost:2345 \
143+
--url=http://localhost:${DLE_SERVER_PORT} \
116144
--token=secret_token \
117145
--insecure
118146

@@ -128,14 +156,14 @@ dblab clone create \
128156

129157
# Connect to a clone and check the available table
130158
PGPASSWORD=secret_password psql \
131-
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
159+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
132160

133161
# Drop table
134162
PGPASSWORD=secret_password psql \
135-
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c 'drop table pgbench_accounts'
163+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c 'drop table pgbench_accounts'
136164

137165
PGPASSWORD=secret_password psql \
138-
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
166+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
139167

140168
## Reset clone
141169
dblab clone reset testclone
@@ -145,11 +173,14 @@ dblab clone status testclone
145173

146174
# Check the database objects (everything should be the same as when we started)
147175
PGPASSWORD=secret_password psql \
148-
"host=localhost port=6000 user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
176+
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
149177

150178
### Step 4. Destroy clone
151179
dblab clone destroy testclone
152180
dblab clone list
153181

182+
## Stop DLE.
183+
sudo docker stop ${DLE_SERVER_NAME}
184+
154185
### Finish. clean up
155186
source "${DIR}/_cleanup.sh"

0 commit comments

Comments
 (0)