K8SPSMDB-1308: Improve physical restore logs (#1915)

* K8SPSMDB-1308: Improve physical restore logs

* K8SPSMDB-1265: Update versions for 1.20.0 release (#1903)

* K8SPSMDB-1265: Update versions for 1.20.0 release

* Update test dependencies versions

* Update cr images

* Update Mongo 8 image to 8.0.8-3

* Fix compare files with correct PBM_MONGODB_URI (#1905)

* K8SPSMDB-1265: update oc compare for rs-shard-migration (#1909)

* K8SPSMDB-1265: clean up basmdb-backup and use kubectl wait for wait_restore (#1911)

Use kubectl wait instead of regular loop in `wait_restore()`
Add retry for `demand-backup-sharded` test backup presence in minio storage
Delete backups during test cleanup before removing finalizers from objects.

* Revert to use Mongo 7 as default

* Print operator version (#1921)

* K8SPSMDB-1265: Print operator version details

* fix one-pod on azure

* fix balancer on aks

* K8SPSMDB-1268 pmm3 support (#1916)

* K8SPSMDB-1268 pmm3 support

* fix imports

* assert the env var lengths

* add server host

* cover more inits in test

* add custom params

* improve test

* remove spammy logs

* e2e tests

* fix mounts for pmm3 container

* update secret with new token comment

* wrapup e2e test

* drop unused env vars

* ensure that pmm3 test is fully functional

* bonus: improve the custom name e2e verification

* add small assertion to ensure that disabled pmm and nil secret return no container

* make custom cluster name configurable in cr for the e2e test

* fix linter

* cr: package rename to config

* add some more test cases

* K8SPSMDB-1216 update to db.hello (#1929)

* K8SPSMDB-1216 update to db.hello

* update rs-shard-migration test

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
Co-authored-by: Julio Pasinatto <julio.pasinatto@percona.com>
Co-authored-by: Eleonora Zinchenko <eleonora.zinchenko@percona.com>
Co-authored-by: George Kechagias <geo.kechagias@gmail.com>
Co-authored-by: Natalia Marukovich <nmarukovich@gmail.com>

Author: 6 people

build/physical-restore-ps-entry.sh

@@ -1,15 +1,36 @@
 #!/bin/bash
 
-set -Eeuo pipefail
+set -e
 set -o xtrace
 
-log=/tmp/pbm-agent.log
+PBM_AGENT_LOG=/tmp/pbm-agent.log
+MONGOD_LOG=/tmp/mongod.log
+PHYSICAL_RESTORE_DIR=/data/db/pbm-restore-logs
+
+function handle_sigterm() {
+	echo "Received SIGTERM, cleaning up..."
+
+	mkdir ${PHYSICAL_RESTORE_DIR}
+	mv pbm.restore.log.* ${PBM_AGENT_LOG} ${MONGOD_LOG} ${PHYSICAL_RESTORE_DIR}/
+
+	echo "Restore finished, you can find logs in ${PHYSICAL_RESTORE_DIR}"
+	exit 0
+}
+
+trap 'handle_sigterm' 15
 
 touch /opt/percona/restore-in-progress
 
-/opt/percona/pbm-agent 1>&2 2>${log} &
-/opt/percona/ps-entry.sh "$@" 1>&2 2>/tmp/mongod.log
+/opt/percona/pbm-agent >${PBM_AGENT_LOG} 2>&1 &
+pbm_pid=$!
+
+/opt/percona/ps-entry.sh "$@" >${MONGOD_LOG} 2>&1 &
+mongod_pid=$!
 
-echo "Physical restore in progress"
-tail -n +1 -f ${log}
-sleep infinity
+set +o xtrace
+echo "Physical restore in progress... pbm-agent logs: ${PBM_AGENT_LOG} mongod logs: ${MONGOD_LOG}"
+echo "Script PID: $$, pbm-agent PID: $pbm_pid, mongod PID: $mongod_pid"
+while true; do
+    echo "Still in progress at $(date)"
+    sleep 10
+done

e2e-tests/functions

@@ -343,6 +343,36 @@ simple_data_check() {
 	fi
 }
 
+get_mongod_pods() {
+	local cluster=$1
+
+	kubectl_bin get pod \
+		--no-headers \
+		-l app.kubernetes.io/instance=${cluster} \
+		-l app.kubernetes.io/component=mongod
+
+}
+
+collect_physical_restore_logs() {
+	local cluster=$1
+	local restore=$2
+
+	for pod in $(get_mongod_pods ${cluster}); do
+		desc "pbm-agent logs from ${pod}"
+		kubectl_bin exec -it ${pod} -- cat /tmp/pbm-agent.log || true
+	done
+}
+
+is_physical_backup() {
+	local backup=$1
+
+	if [[ $(kubectl_bin get psmdb-backup ${backup} -o jsonpath={.status.type}) == "physical" ]]; then
+		return 0
+	fi
+
+	return 1
+}
+
 wait_restore() {
 	local backup_name=$1
 	local cluster_name=$2
@@ -384,6 +414,11 @@ wait_restore() {
 				| grep -v 'Getting tasks for pod' \
 				| grep -v 'Getting pods from source' \
 				| tail -100
+
+			if is_physical_backup ${backup_name}; then
+				collect_physical_restore_logs
+			fi
+
 			kubectl_bin get psmdb-restore restore-${backup_name} -o yaml
 			kubectl_bin describe psmdb-restore restore-${backup_name}
 			echo "Restore object restore-${backup_name} is in ${current_state} state."