Skip to content

Commit 79acecd

Browse files
vitabaksvitabaks
committed
Merge branch 'database-properly-shutdown' into 'master'
tests(engine): check that database system was properly shut down (#350) See merge request postgres-ai/database-lab!551
2 parents 6883350 + e884ca4 commit 79acecd

File tree

6 files changed

+168
-14
lines changed

6 files changed

+168
-14
lines changed

‎engine/configs/standard/postgres/default/9.6/postgresql.dblab.postgresql.conf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ dynamic_shared_memory_type = posix # the default is the first option
341341
# (change requires restart)
342342

343343
# These are only used if logging_collector is on:
344-
#log_directory = 'pg_log' # directory where log files are written,
344+
log_directory = 'log' # directory where log files are written,
345345
# can be absolute or relative to PGDATA
346346
#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern,
347347
# can include strftime() escapes

‎engine/test/1.synthetic.sh‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
1010
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
1111
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
1212
export DLE_PORT_POOL_TO=${DLE_PORT_POOL_TO:-9100}
13-
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
13+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
1414
export DLE_TEST_POOL_NAME="test_dblab_pool"
1515

1616
DIR=${0%/*}
@@ -91,9 +91,12 @@ yq eval -i '
9191
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
9292
' "${configDir}/server.yml"
9393

94-
# logerrors is not supported in PostgreSQL 9.6
94+
# Edit the following options for PostgreSQL 9.6
9595
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
96-
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
96+
yq eval -i '
97+
.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain" |
98+
.databaseConfigs.configs.log_directory = "log"
99+
' "${configDir}/server.yml"
97100
fi
98101

99102
## Launch Database Lab server
@@ -161,6 +164,21 @@ dblab clone create \
161164
--password secret_password \
162165
--id testclone
163166

167+
### Check that database system was properly shut down (clone data dir)
168+
CLONE_LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/clones/dblab_clone_"${DLE_PORT_POOL_FROM}"/data/log
169+
LOG_FILE_CSV=$(sudo ls -t "$CLONE_LOG_DIR" | grep .csv | head -n 1)
170+
if sudo test -d "$CLONE_LOG_DIR"
171+
then
172+
if sudo grep -q 'database system was not properly shut down; automatic recovery in progress' "$CLONE_LOG_DIR"/"$LOG_FILE_CSV"
173+
then
174+
echo "ERROR: database system was not properly shut down" && exit 1
175+
else
176+
echo "INFO: database system was properly shut down - OK"
177+
fi
178+
else
179+
echo "ERROR: the log directory \"$CLONE_LOG_DIR\" does not exist" && exit 1
180+
fi
181+
164182
# Connect to a clone and check the available table
165183
PGPASSWORD=secret_password psql \
166184
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'

‎engine/test/2.logical_generic.sh‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,12 @@ yq eval -i '
105105
.databaseContainer.dockerImage = "postgresai/extended-postgres:" + strenv(POSTGRES_VERSION)
106106
' "${configDir}/server.yml"
107107

108-
# logerrors is not supported in PostgreSQL 9.6
108+
# Edit the following options for PostgreSQL 9.6
109109
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
110-
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
110+
yq eval -i '
111+
.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain" |
112+
.databaseConfigs.configs.log_directory = "log"
113+
' "${configDir}/server.yml"
111114
fi
112115

113116
## Launch Database Lab server
@@ -171,6 +174,21 @@ dblab clone create \
171174
--password secret_password \
172175
--id testclone
173176

177+
### Check that database system was properly shut down (clone data dir)
178+
CLONE_LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/clones/dblab_clone_"${DLE_PORT_POOL_FROM}"/data/log
179+
LOG_FILE_CSV=$(sudo ls -t "$CLONE_LOG_DIR" | grep .csv | head -n 1)
180+
if sudo test -d "$CLONE_LOG_DIR"
181+
then
182+
if sudo grep -q 'database system was not properly shut down; automatic recovery in progress' "$CLONE_LOG_DIR"/"$LOG_FILE_CSV"
183+
then
184+
echo "ERROR: database system was not properly shut down" && exit 1
185+
else
186+
echo "INFO: database system was properly shut down - OK"
187+
fi
188+
else
189+
echo "ERROR: the log directory \"$CLONE_LOG_DIR\" does not exist" && exit 1
190+
fi
191+
174192
# Connect to a clone and check the available table
175193
PGPASSWORD=secret_password psql \
176194
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
@@ -199,5 +217,20 @@ dblab clone list
199217
## Stop DLE.
200218
sudo docker stop ${DLE_SERVER_NAME}
201219

220+
### Check that database system was properly shut down (main data dir)
221+
LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/data/log
222+
LOG_FILE_CSV=$(sudo ls -t "$LOG_DIR" | grep .csv | head -n 1)
223+
if sudo test -d "$LOG_DIR"
224+
then
225+
if [[ $(sudo tail -n 10 "$LOG_DIR"/"$LOG_FILE_CSV" | grep -c 'received fast shutdown request\|database system is shut down') = 2 ]]
226+
then
227+
echo "INFO: database system was properly shut down - OK"
228+
else
229+
echo "ERROR: database system was not properly shut down" && exit 1
230+
fi
231+
else
232+
echo "ERROR: the log directory \"$LOG_DIR\" does not exist" && exit 1
233+
fi
234+
202235
### Finish. clean up
203236
source "${DIR}/_cleanup.sh"

‎engine/test/3.physical_walg.sh‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ DLE_SERVER_NAME="dblab_server_test"
88
# Environment variables for replacement rules
99
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
1010
export WALG_BACKUP_NAME="${WALG_BACKUP_NAME:-"LATEST"}"
11-
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
11+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
1212
export DLE_TEST_POOL_NAME="test_dblab_pool"
1313
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
1414
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -58,9 +58,14 @@ yq eval -i '
5858
.retrieval.spec.physicalSnapshot.options.skipStartSnapshot = true
5959
' "${configDir}/server.yml"
6060

61-
# logerrors is not supported in PostgreSQL 9.6
61+
# Edit the following options for PostgreSQL 9.6
6262
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
63-
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
63+
yq eval -i '
64+
.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain" |
65+
.databaseConfigs.configs.log_directory = "log" |
66+
.retrieval.spec.physicalRestore.options.sync.configs.log_directory = "log" |
67+
.retrieval.spec.physicalSnapshot.options.promotion.configs.log_directory = "log"
68+
' "${configDir}/server.yml"
6469
fi
6570

6671
set +euxo pipefail # ---- do not display secrets
@@ -177,6 +182,21 @@ dblab clone create \
177182
--password secret_password \
178183
--id testclone
179184

185+
### Check that database system was properly shut down (clone data dir)
186+
CLONE_LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/clones/dblab_clone_"${DLE_PORT_POOL_FROM}"/data/log
187+
LOG_FILE_CSV=$(sudo ls -t "$CLONE_LOG_DIR" | grep .csv | head -n 1)
188+
if sudo test -d "$CLONE_LOG_DIR"
189+
then
190+
if sudo grep -q 'database system was not properly shut down; automatic recovery in progress' "$CLONE_LOG_DIR"/"$LOG_FILE_CSV"
191+
then
192+
echo "ERROR: database system was not properly shut down" && exit 1
193+
else
194+
echo "INFO: database system was properly shut down - OK"
195+
fi
196+
else
197+
echo "ERROR: the log directory \"$CLONE_LOG_DIR\" does not exist" && exit 1
198+
fi
199+
180200
PGPASSWORD=secret_password psql \
181201
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c 'show max_wal_senders'
182202

@@ -208,6 +228,21 @@ dblab clone list
208228
## Stop DLE.
209229
sudo docker stop ${DLE_SERVER_NAME}
210230

231+
### Check that database system was properly shut down (main data dir)
232+
LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/data/log
233+
LOG_FILE_CSV=$(sudo ls -t "$LOG_DIR" | grep .csv | head -n 1)
234+
if sudo test -d "$LOG_DIR"
235+
then
236+
if [[ $(sudo tail -n 10 "$LOG_DIR"/"$LOG_FILE_CSV" | grep -c 'received fast shutdown request\|database system is shut down') = 2 ]]
237+
then
238+
echo "INFO: database system was properly shut down - OK"
239+
else
240+
echo "ERROR: database system was not properly shut down" && exit 1
241+
fi
242+
else
243+
echo "ERROR: the log directory \"$LOG_DIR\" does not exist" && exit 1
244+
fi
245+
211246
## Stop control containers.
212247
cleanup_service_containers
213248

‎engine/test/4.physical_basebackup.sh‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,14 @@ yq eval -i '
122122
.retrieval.spec.physicalRestore.options.customTool.command = "pg_basebackup -X stream -D " + strenv(DLE_TEST_MOUNT_DIR) + "/" + strenv(DLE_TEST_POOL_NAME) + "/data"
123123
' "${configDir}/server.yml"
124124

125-
# logerrors is not supported in PostgreSQL 9.6
125+
# Edit the following options for PostgreSQL 9.6
126126
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
127-
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
127+
yq eval -i '
128+
.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain" |
129+
.databaseConfigs.configs.log_directory = "log" |
130+
.retrieval.spec.physicalRestore.options.sync.configs.log_directory = "log" |
131+
.retrieval.spec.physicalSnapshot.options.promotion.configs.log_directory = "log"
132+
' "${configDir}/server.yml"
128133
fi
129134

130135
## Launch Database Lab server
@@ -193,6 +198,21 @@ dblab clone create \
193198
--password secret_password \
194199
--id testclone
195200

201+
### Check that database system was properly shut down (clone data dir)
202+
CLONE_LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/clones/dblab_clone_"${DLE_PORT_POOL_FROM}"/data/log
203+
LOG_FILE_CSV=$(sudo ls -t "$CLONE_LOG_DIR" | grep .csv | head -n 1)
204+
if sudo test -d "$CLONE_LOG_DIR"
205+
then
206+
if sudo grep -q 'database system was not properly shut down; automatic recovery in progress' "$CLONE_LOG_DIR"/"$LOG_FILE_CSV"
207+
then
208+
echo "ERROR: database system was not properly shut down" && exit 1
209+
else
210+
echo "INFO: database system was properly shut down - OK"
211+
fi
212+
else
213+
echo "ERROR: the log directory \"$CLONE_LOG_DIR\" does not exist" && exit 1
214+
fi
215+
196216
# Connect to a clone and check the available table
197217
PGPASSWORD=secret_password psql \
198218
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=test" -c '\dt+'
@@ -221,6 +241,21 @@ dblab clone list
221241
## Stop DLE.
222242
sudo docker stop ${DLE_SERVER_NAME}
223243

244+
### Check that database system was properly shut down (main data dir)
245+
LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/data/log
246+
LOG_FILE_CSV=$(sudo ls -t "$LOG_DIR" | grep .csv | head -n 1)
247+
if sudo test -d "$LOG_DIR"
248+
then
249+
if [[ $(sudo tail -n 10 "$LOG_DIR"/"$LOG_FILE_CSV" | grep -c 'received fast shutdown request\|database system is shut down') = 2 ]]
250+
then
251+
echo "INFO: database system was properly shut down - OK"
252+
else
253+
echo "ERROR: database system was not properly shut down" && exit 1
254+
fi
255+
else
256+
echo "ERROR: the log directory \"$LOG_DIR\" does not exist" && exit 1
257+
fi
258+
224259
## Stop control containers.
225260
cleanup_service_containers
226261

‎engine/test/5.logical_rds.sh‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ IMAGE2TEST="registry.gitlab.com/postgres-ai/database-lab/dblab-server:${TAG}"
66
DLE_SERVER_NAME="dblab_server_test"
77

88
# Environment variables for replacement rules
9-
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
9+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
1010
export DLE_TEST_POOL_NAME="test_dblab_pool"
1111
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
1212
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -57,9 +57,12 @@ yq eval -i '
5757
.retrieval.spec.logicalRestore.options.dumpLocation = env(DLE_TEST_MOUNT_DIR) + "/" + env(DLE_TEST_POOL_NAME) + "/dump"
5858
' "${configDir}/server.yml"
5959

60-
# logerrors is not supported in PostgreSQL 9.6
60+
# Edit the following options for PostgreSQL 9.6
6161
if [ "${POSTGRES_VERSION}" = "9.6" ]; then
62-
yq eval -i '.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain"' "${configDir}/server.yml"
62+
yq eval -i '
63+
.databaseConfigs.configs.shared_preload_libraries = "pg_stat_statements, auto_explain" |
64+
.databaseConfigs.configs.log_directory = "log"
65+
' "${configDir}/server.yml"
6366
fi
6467

6568
# Download AWS RDS certificate
@@ -130,6 +133,21 @@ dblab clone create \
130133
--password secret_password \
131134
--id testclone
132135

136+
### Check that database system was properly shut down (clone data dir)
137+
CLONE_LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/clones/dblab_clone_"${DLE_PORT_POOL_FROM}"/data/log
138+
LOG_FILE_CSV=$(sudo ls -t "$CLONE_LOG_DIR" | grep .csv | head -n 1)
139+
if sudo test -d "$CLONE_LOG_DIR"
140+
then
141+
if sudo grep -q 'database system was not properly shut down; automatic recovery in progress' "$CLONE_LOG_DIR"/"$LOG_FILE_CSV"
142+
then
143+
echo "ERROR: database system was not properly shut down" && exit 1
144+
else
145+
echo "INFO: database system was properly shut down - OK"
146+
fi
147+
else
148+
echo "ERROR: the log directory \"$CLONE_LOG_DIR\" does not exist" && exit 1
149+
fi
150+
133151
# Connect to a clone and check the available table
134152
PGPASSWORD=secret_password psql \
135153
"host=localhost port=${DLE_PORT_POOL_FROM} user=dblab_user_1 dbname=${SOURCE_DBNAME}" -c '\dt+'
@@ -158,5 +176,20 @@ dblab clone list
158176
## Stop DLE.
159177
sudo docker stop ${DLE_SERVER_NAME}
160178

179+
### Check that database system was properly shut down (main data dir)
180+
LOG_DIR="${DLE_TEST_MOUNT_DIR}"/"${DLE_TEST_POOL_NAME}"/data/log
181+
LOG_FILE_CSV=$(sudo ls -t "$LOG_DIR" | grep .csv | head -n 1)
182+
if sudo test -d "$LOG_DIR"
183+
then
184+
if [[ $(sudo tail -n 10 "$LOG_DIR"/"$LOG_FILE_CSV" | grep -c 'received fast shutdown request\|database system is shut down') = 2 ]]
185+
then
186+
echo "INFO: database system was properly shut down - OK"
187+
else
188+
echo "ERROR: database system was not properly shut down" && exit 1
189+
fi
190+
else
191+
echo "ERROR: the log directory \"$LOG_DIR\" does not exist" && exit 1
192+
fi
193+
161194
### Finish. clean up
162195
source "${DIR}/_cleanup.sh"

0 commit comments

Comments
 (0)