Add collect_k8s_logs into tests and add pod describe

Author: tplavcic

e2e-tests/arbiter/run

@@ -31,6 +31,7 @@ check_cr_config() {
 	if [[ $(kubectl_bin get pod \
 		--selector=statefulset.kubernetes.io/pod-name="${cluster}-arbiter-0" \
 		-o jsonpath='{.items[*].status.containerStatuses[?(@.name == "mongod-arbiter")].restartCount}') -gt 0 ]]; then
+		collect_k8s_logs
 		echo "Something went wrong with arbiter. Exiting..."
 		exit 1
 	fi

e2e-tests/balancer/run

@@ -15,6 +15,7 @@ check_balancer() {
 		| grep -E -v "Percona Server for MongoDB|connecting to:|Implicit session:|versions do not match|Error saving history file:|bye")
 
 	if [[ $balancer_running != "$expected" ]]; then
+		collect_k8s_logs
 		echo "Unexpected output from \"db.adminCommand({balancerStatus: 1}).mode\": $balancer_running"
 		echo "Expected $expected"
 		exit 1

e2e-tests/cross-site-sharded/run

@@ -101,6 +101,7 @@ for i in "rs0" "rs1"; do
 done
 
 if [[ $shards -lt 2 ]]; then
+	collect_k8s_logs
 	echo "data is only on some of the shards, maybe sharding is not working"
 	exit 1
 fi

e2e-tests/data-at-rest-encryption/run

@@ -83,6 +83,7 @@ encrypted_cluster_log=$(kubectl_bin logs some-name-rs0-0 -c mongod -n $namespace
 
 echo "$encrypted_cluster_log"
 if [ -z "$encrypted_cluster_log" ]; then
+	collect_k8s_logs
 	echo "Cluster is not encrypted"
 	exit 1
 fi
@@ -99,6 +100,7 @@ until [ "$retry" -ge 10 ]; do
 		echo "Cluster is not encrypted already"
 		break
 	elif [ $retry == 15 ]; then
+		collect_k8s_logs
 		echo "Max retry count $retry reached. Cluster is still encrypted"
 		exit 1
 	else

e2e-tests/data-sharded/run

@@ -17,6 +17,7 @@ check_rs_proper_component_deletion() {
 	until [[ $(kubectl_bin get sts -l app.kubernetes.io/instance=${cluster},app.kubernetes.io/replset=${rs_name} -ojson | jq '.items | length') -eq 0 ]]; do
 		let retry+=1
 		if [ $retry -ge 70 ]; then
+			collect_k8s_logs
 			sts_count=$(kubectl_bin get sts -l app.kubernetes.io/instance=${cluster},app.kubernetes.io/replset=${rs_name} -ojson | jq '.items | length')
 			echo "Replset $rs_name not properly removed, expected sts count of 0 but got $sts_count. Exiting after $retry tries..."
 			exit 1
@@ -115,6 +116,7 @@ main() {
 	done
 
 	if [[ $shards -lt 3 ]]; then
+		collect_k8s_logs
 		echo "data is only on some of the shards, maybe sharding is not working"
 		exit 1
 	fi
@@ -125,6 +127,7 @@ main() {
 		"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
 		"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls")
 	if ! echo $res | grep -q '"ok" : 1'; then
+		collect_k8s_logs
 		echo "app database not dropped. Exiting.."
 		exit 1
 	fi

e2e-tests/default-cr/run

@@ -27,6 +27,7 @@ function stop_cluster() {
 		let passed_time="${passed_time}+${sleep_time}"
 		sleep ${sleep_time}
 		if [[ ${passed_time} -gt ${max_wait_time} ]]; then
+			collect_k8s_logs
 			echo "We've been waiting for cluster stop for too long. Exiting..."
 			exit 1
 		fi

e2e-tests/demand-backup-physical-sharded/run

@@ -38,6 +38,7 @@ run_recovery_check() {
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
+		collect_k8s_logs
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
 		exit 1
 	fi
@@ -52,6 +53,7 @@ check_exported_mongos_service_endpoint() {
 	local host=$1
 
 	if [ "$host" != "$(kubectl_bin get psmdb $cluster -o=jsonpath='{.status.host}')" ]; then
+		collect_k8s_logs
 		echo "Exported host is not correct after the restore"
 		exit 1
 	fi
@@ -80,6 +82,7 @@ wait_cluster_consistency ${cluster}
 lbEndpoint=$(kubectl_bin get svc $cluster-mongos -o=jsonpath='{.status}' |
 	jq -r 'select(.loadBalancer != null and .loadBalancer.ingress != null and .loadBalancer.ingress != []) | .loadBalancer.ingress[0][]')
 if [ -z $lbEndpoint ]; then
+	collect_k8s_logs
 	echo "mongos service not exported correctly"
 	exit 1
 fi

e2e-tests/demand-backup-physical/run

@@ -38,6 +38,7 @@ run_recovery_check() {
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
+		collect_k8s_logs
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
 		exit 1
 	fi

e2e-tests/demand-backup-sharded/run

@@ -166,6 +166,7 @@ backup_exists=$(kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --rest
 	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ \
 	| grep -c ${backup_dest_minio}_ | cat)
 if [[ $backup_exists -eq 1 ]]; then
+	collect_k8s_logs
 	echo "Backup was not removed from bucket -- minio"
 	exit 1
 fi

e2e-tests/demand-backup/run

@@ -135,6 +135,7 @@ backup_exists=$(kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --rest
 	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ \
 	| grep -c ${backup_dest_minio} | cat)
 if [[ $backup_exists -eq 1 ]]; then
+	collect_k8s_logs
 	echo "Backup was not removed from bucket -- minio"
 	exit 1
 fi
@@ -170,6 +171,7 @@ backup_exists=$(kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --rest
 	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls s3://operator-testing/ \
 	| grep -c ${backup_dest_minio} | cat)
 if [[ $backup_exists -eq 1 ]]; then
+	collect_k8s_logs
 	echo "Backup was not removed from bucket -- minio"
 	exit 1
 fi