roachtest: skip latency verification for 2 mins after getting to steady state

This patch modifies the latency verifier to sleep for 2 minutes after detecting
steady state (i.e. catchup scan/ initial scan completion), so a test doesn't
flake after a temporary post catchup scan latency blip.

Informs #156707

Release note: none

Author: msbutler

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -1205,6 +1205,7 @@ func (rd *replicationDriver) main(ctx context.Context) {
 		if err := lv.pollLatencyUntilJobSucceeds(ctx, rd.setup.dst.db, ingestionJobID, time.Second, workloadDoneCh); err != nil {
 			// The latency poller may have failed because latency got too high. Grab a
 			// debug zip before the replication jobs spin down.
+			rd.t.L().Printf("latency monitor detected an error: %s", err)
 			rd.fetchDebugZip(ctx, rd.setup.src.nodes, "latency_source_debug.zip")
 			rd.fetchDebugZip(ctx, rd.setup.dst.nodes, "latency_dest_debug.zip")
 			return err

pkg/cmd/roachtest/tests/latency_verifier.go

@@ -115,8 +115,11 @@ func (lv *latencyVerifier) noteHighwater(highwaterTime time.Time) {
 		}
 		return
 	}
-
+	justSwitchedToSteadyState := false
 	if lv.targetSteadyLatency == 0 || latency < lv.targetSteadyLatency {
+		if !lv.latencyBecameSteady {
+			justSwitchedToSteadyState = true
+		}
 		lv.latencyBecameSteady = true
 	}
 	if !lv.latencyBecameSteady {
@@ -147,6 +150,10 @@ func (lv *latencyVerifier) noteHighwater(highwaterTime time.Time) {
 			lv.name, latency.Truncate(time.Millisecond), lv.maxSeenSteadyLatency.Truncate(time.Millisecond), highwaterTime)
 		lv.setTestStatus(update)
 	}
+	if justSwitchedToSteadyState {
+		lv.logger.Printf("just reached steady state, sleeping for 2 minutes to ensure no temporary latency regressions")
+		time.Sleep(2 * time.Minute)
+	}
 }
 
 // pollLatencyUntilJobSucceeds polls the changefeed latency until it is