Improve e2e test suite retries (#2387)

Author: johscheuer

e2e/fixtures/fdb_cluster.go

@@ -1245,6 +1245,7 @@ func (fdbCluster *FdbCluster) SetKillProcesses(allowKill bool, wait bool) {
 // set to true only this condition is allowed.
 func (fdbCluster *FdbCluster) AllProcessGroupsHaveCondition(
 	condition fdbv1beta2.ProcessGroupConditionType,
+	ignoreMissing bool,
 ) bool {
 	cluster := fdbCluster.GetCluster()
 
@@ -1253,6 +1254,10 @@ func (fdbCluster *FdbCluster) AllProcessGroupsHaveCondition(
 			continue
 		}
 
+		if processGroup.GetConditionTime(fdbv1beta2.MissingProcesses) != nil && ignoreMissing {
+			continue
+		}
+
 		if len(processGroup.ProcessGroupConditions) != 1 {
 			return false
 		}
@@ -1851,13 +1856,15 @@ func (fdbCluster *FdbCluster) WriteKeyValue(
 	keyValue KeyValue,
 	timeout int,
 ) {
-	_, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(
-		fmt.Sprintf("writemode on; set %s %s", keyValue.GetKey(), keyValue.GetValue()),
-		false,
-		timeout,
-	)
+	gomega.Eventually(func(g gomega.Gomega) {
+		_, stderr, err := fdbCluster.RunFdbCliCommandInOperatorWithoutRetry(
+			fmt.Sprintf("writemode on; set %s %s", keyValue.GetKey(), keyValue.GetValue()),
+			false,
+			timeout,
+		)
 
-	gomega.Expect(err).NotTo(gomega.HaveOccurred(), stderr)
+		g.Expect(err).NotTo(gomega.HaveOccurred(), stderr)
+	}).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(gomega.Succeed())
 }
 
 // WriteKeyValuesWithTimeout writes multiples key values into FDB with the specified timeout.

e2e/test_operator_ha_upgrades/operator_ha_upgrade_test.go

@@ -116,21 +116,19 @@ func clusterSetup(beforeVersion string, enableOperatorPodChaos bool) {
 // are at "IncorrectCommandLine" state (which means that they have the new binaries and have
 // the new configuration but are not restarted).
 //
-// Note: we tried doing this by polling for  "UpgradeRequeued" event on primary/remote/primary-satellite data centers,
+// Note: we tried doing this by polling for "UpgradeRequeued" event on primary/remote/primary-satellite data centers,
 // but we found that that approach is not very reliable.
 func verifyBouncingIsBlocked() {
-	Eventually(func(g Gomega) bool {
+	Eventually(func(g Gomega) {
 		for _, cluster := range fdbCluster.GetAllClusters() {
 			if strings.HasSuffix(cluster.Name(), fixtures.RemoteSatelliteID) {
 				continue
 			}
 
-			g.Expect(cluster.AllProcessGroupsHaveCondition(fdbv1beta2.IncorrectCommandLine)).
-				To(BeTrue())
+			g.Expect(cluster.AllProcessGroupsHaveCondition(fdbv1beta2.IncorrectCommandLine, true)).
+				To(BeTrue(), fmt.Sprintf("all process groups in cluster %s should have the incorrect command line condition", cluster.Name()))
 		}
-
-		return true
-	}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(30).Should(BeTrue())
+	}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(30).Should(Succeed())
 }
 
 var _ = AfterSuite(func() {
@@ -335,7 +333,7 @@ var _ = Describe("Operator HA Upgrades", Label("e2e", "pr"), func() {
 
 			// It should block the restart of the processes until the rest of the cluster is upgraded.
 			Eventually(func() bool {
-				return primary.AllProcessGroupsHaveCondition(fdbv1beta2.IncorrectCommandLine)
+				return primary.AllProcessGroupsHaveCondition(fdbv1beta2.IncorrectCommandLine, true)
 			}).WithTimeout(10 * time.Minute).WithPolling(5 * time.Second).MustPassRepeatedly(30).Should(BeTrue())
 
 			Expect(fdbCluster.UpgradeCluster(targetVersion, false)).NotTo(HaveOccurred())

e2e/test_operator_upgrades/operator_upgrades_test.go

@@ -217,7 +217,10 @@ var _ = Describe("Operator Upgrades", Label("e2e", "pr"), func() {
 
 			// Wait until all process groups are in the staging phase and the new binaries are available.
 			Eventually(func() bool {
-				return fdbCluster.AllProcessGroupsHaveCondition(fdbv1beta2.IncorrectCommandLine)
+				return fdbCluster.AllProcessGroupsHaveCondition(
+					fdbv1beta2.IncorrectCommandLine,
+					true,
+				)
 			}).WithTimeout(10 * time.Minute).WithPolling(2 * time.Second).Should(BeTrue())
 
 			// Restart the fdbserver process to pickup the new configuration and run with the newer version.