roachtest: use decommission nudger for decommissionbench.go

This commit adds the decommissioning nudger as a new flavor in
`decommissionbench.go` and updates them to run weekly instead.

This commit also removes the metamorphically enabled decommissioning nudger from
`runDecommissionSlow`, since these roachtests typically complete in under 10
minutes and usually pass. The `decommissionBench`, on the other hand, runs
longer and is a better fit for this configuration.

Author: wenyihu6

pkg/cmd/roachtest/tests/decommission.go

@@ -1135,19 +1135,6 @@ func runDecommissionSlow(ctx context.Context, t test.Test, c cluster.Cluster) {
 		// Increase the speed of decommissioning.
 		run(db, `SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='2GiB'`)
 
-		// Metamorphically enable the decommissioning nudger to get more test
-		// coverage on decommissioning nudger.
-		{
-			seed := timeutil.Now().UnixNano()
-			t.L().Printf("seed: %d", seed)
-			rng := rand.New(rand.NewSource(seed))
-
-			if rng.Intn(2) == 0 {
-				run(db, `SET CLUSTER SETTING kv.enqueue_in_replicate_queue_on_problem.interval = '10m'`)
-				t.L().Printf("metamorphically enabled decommissioning nudger")
-			}
-		}
-
 		// Wait for initial up-replication.
 		err := roachtestutil.WaitForReplication(ctx, t.L(), db, replicationFactor, roachprod.AtLeastReplicationFactor)
 		require.NoError(t, err)

pkg/cmd/roachtest/tests/decommissionbench.go

@@ -89,6 +89,12 @@ type decommissionBenchSpec struct {
 	// instead of a random node.
 	decommissionNode int
 
+	// When true, the decommissioning nudger
+	// (kv.enqueue_in_replicate_queue_on_problem.interval) will be used to
+	// periodically enqueue decommissioning ranges at its leaseholders every 5
+	// minutes.
+	useDecommissioningNudger bool
+
 	skip string
 }
 
@@ -198,6 +204,16 @@ func registerDecommissionBench(r registry.Registry) {
 			multiregion:        true,
 			decommissionNode:   2,
 		},
+		{
+			// Same as above but using the decommissioning nudger.
+			nodes:                    6,
+			warehouses:               1000,
+			whileUpreplicating:       true,
+			drainFirst:               true,
+			multiregion:              true,
+			decommissionNode:         2,
+			useDecommissioningNudger: true,
+		},
 		{
 			// Multiregion decommission, and add a new node in a different region.
 			nodes:              6,
@@ -207,6 +223,16 @@ func registerDecommissionBench(r registry.Registry) {
 			multiregion:        true,
 			decommissionNode:   3,
 		},
+		{
+			// Same as above but using the decommissioning nudger.
+			nodes:                    6,
+			warehouses:               1000,
+			whileUpreplicating:       true,
+			drainFirst:               true,
+			multiregion:              true,
+			decommissionNode:         3,
+			useDecommissioningNudger: true,
+		},
 	} {
 		registerDecommissionBenchSpec(r, benchSpec)
 	}
@@ -274,6 +300,10 @@ func registerDecommissionBenchSpec(r registry.Registry, benchSpec decommissionBe
 		extraNameParts = append(extraNameParts, "multi-region")
 	}
 
+	if benchSpec.useDecommissioningNudger {
+		extraNameParts = append(extraNameParts, "use-nudger")
+	}
+
 	// Save some money and CPU quota by using a smaller workload CPU. Only
 	// do this for cluster of size 3 or smaller to avoid regressions.
 	specOptions = append(specOptions, spec.WorkloadNode())
@@ -299,7 +329,7 @@ func registerDecommissionBenchSpec(r registry.Registry, benchSpec decommissionBe
 			specOptions...,
 		),
 		CompatibleClouds:    registry.OnlyGCE,
-		Suites:              registry.Suites(registry.Nightly),
+		Suites:              registry.Suites(registry.Weekly),
 		SkipPostValidations: registry.PostValidationNoDeadNodes,
 		Timeout:             timeout,
 		NonReleaseBlocker:   true,
@@ -427,6 +457,15 @@ func setupDecommissionBench(
 		db := c.Conn(ctx, t.L(), pinnedNode)
 		defer db.Close()
 
+		// Metamorphically enable the decommissioning nudger to get more test
+		// coverage on decommissioning nudger.
+		if benchSpec.useDecommissioningNudger {
+			if _, err := db.ExecContext(ctx, `SET CLUSTER SETTING kv.enqueue_in_replicate_queue_on_problem.interval = '5m'`); err != nil {
+				t.Fatal(err)
+			}
+			t.L().Printf("enabled decommissioning nudger")
+		}
+
 		// Note that we are waiting for 3 replicas only. We can't assume 5 replicas
 		// here because 5 only applies to system ranges so we will never reach this
 		// number globally. We also don't know if all upreplication succeeded, but