K8SPSMDB-993: increase timeout for waiting pbm resync (#1512)

* K8SPSMDB-993: increase timeout for waiting pbm resync

https://perconadev.atlassian.net/browse/K8SPSMDB-993

* fix tests

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: pooknull

e2e-tests/demand-backup-physical-sharded/run

@@ -35,7 +35,7 @@ run_recovery_check() {
 	compare_kubectl "statefulset/${cluster}-rs0" ${compare_suffix}
 
 	# we don't wait for cluster readiness here because the annotation gets removed then
-	wait_restore "${backup_name}" "${cluster}" "ready" "0" "900"
+	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1600"
 	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"

e2e-tests/demand-backup-physical/run

@@ -35,7 +35,7 @@ run_recovery_check() {
 	compare_kubectl "statefulset/${cluster}-rs0" ${compare_suffix}
 
 	# we don't wait for cluster readiness here because the annotation gets removed then
-	wait_restore "${backup_name}" "${cluster}" "ready" "0" "900"
+	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1600"
 	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"

e2e-tests/pitr-physical/run

@@ -106,7 +106,7 @@ check_recovery() {
 	# fail faster if we don't reach requested status until some time
 	wait_restore "$backup_name" "$cluster_name" "requested" "0" "900"
 	echo
-	wait_restore "$backup_name" "$cluster_name" "ready" "0" "900"
+	wait_restore "$backup_name" "$cluster_name" "ready" "0" "1600"
 	echo
 	set -o xtrace
 

pkg/controller/perconaservermongodbrestore/physical.go

@@ -162,54 +162,55 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont
 			return status, errors.Wrapf(err, "resync config stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String())
 		}
 
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-
-		timeout := time.NewTimer(900 * time.Second)
-		defer timeout.Stop()
-
-	outer:
-		for {
-			select {
-			case <-timeout.C:
-				return status, errors.Errorf("timeout while waiting PBM operation to finish")
-			case <-ticker.C:
-				err := retry.OnError(retry.DefaultBackoff, func(err error) bool { return strings.Contains(err.Error(), "No agent available") }, func() error {
-					stdoutBuf.Reset()
-					stderrBuf.Reset()
-
-					command := []string{"/opt/percona/pbm", "status", "--out", "json"}
-					err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false)
-					if err != nil {
-						log.Error(err, "failed to get PBM status")
-						return err
-					}
-
-					log.V(1).Info("PBM status", "status", stdoutBuf.String())
-
-					return nil
-				})
+		time.Sleep(5 * time.Second) // wait until pbm will start resync
+
+		waitErr := errors.New("waiting for PBM operation to finish")
+		err = retry.OnError(wait.Backoff{
+			Duration: 5 * time.Second,
+			Factor:   2.0,
+			Cap:      time.Hour,
+			Steps:    12,
+		}, func(err error) bool { return err == waitErr }, func() error {
+			err := retry.OnError(retry.DefaultBackoff, func(err error) bool { return strings.Contains(err.Error(), "No agent available") }, func() error {
+				stdoutBuf.Reset()
+				stderrBuf.Reset()
+
+				command := []string{"/opt/percona/pbm", "status", "--out", "json"}
+				err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false)
 				if err != nil {
-					return status, errors.Wrapf(err, "get PBM status stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String())
+					log.Error(err, "failed to get PBM status")
+					return err
 				}
 
-				var pbmStatus struct {
-					Running struct {
-						Type string `json:"type,omitempty"`
-						OpId string `json:"opID,omitempty"`
-					} `json:"running"`
-				}
+				log.V(1).Info("PBM status", "status", stdoutBuf.String())
 
-				if err := json.Unmarshal(stdoutBuf.Bytes(), &pbmStatus); err != nil {
-					return status, errors.Wrap(err, "unmarshal PBM status output")
-				}
+				return nil
+			})
+			if err != nil {
+				return errors.Wrapf(err, "get PBM status stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String())
+			}
 
-				if len(pbmStatus.Running.OpId) == 0 {
-					break outer
-				}
+			var pbmStatus struct {
+				Running struct {
+					Type string `json:"type,omitempty"`
+					OpId string `json:"opID,omitempty"`
+				} `json:"running"`
+			}
 
-				log.Info("Waiting for another PBM operation to finish", "type", pbmStatus.Running.Type, "opID", pbmStatus.Running.OpId)
+			if err := json.Unmarshal(stdoutBuf.Bytes(), &pbmStatus); err != nil {
+				return errors.Wrap(err, "unmarshal PBM status output")
 			}
+
+			if len(pbmStatus.Running.OpId) == 0 {
+				return nil
+			}
+
+			log.Info("Waiting for another PBM operation to finish", "type", pbmStatus.Running.Type, "opID", pbmStatus.Running.OpId)
+
+			return waitErr
+		})
+		if err != nil {
+			return status, err
 		}
 
 		var restoreCommand []string