roachtest: metamorphically enable decommissioning nudger

This commit enables the decommissioning nudger metamorphically for two
roachtests. I selected these two ad hoc — one is labeled as slow, and the other
has been intermittently failing on master to improve observability and help root
casue.

Author: wenyihu6

pkg/cmd/roachtest/tests/decommission.go

@@ -1135,6 +1135,19 @@ func runDecommissionSlow(ctx context.Context, t test.Test, c cluster.Cluster) {
 		// Increase the speed of decommissioning.
 		run(db, `SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='2GiB'`)
 
+		// Metamorphically enable the decommissioning nudger to get more test
+		// coverage on decommissioning nudger.
+		{
+			seed := timeutil.Now().UnixNano()
+			t.L().Printf("seed: %d", seed)
+			rng := rand.New(rand.NewSource(seed))
+
+			if rng.Intn(2) == 0 {
+				run(db, `SET CLUSTER SETTING kv.enqueue_in_replicate_queue_on_problem.interval = '10m'`)
+				t.L().Printf("metamorphically enabled decommissioning nudger")
+			}
+		}
+
 		// Wait for initial up-replication.
 		err := roachtestutil.WaitForReplication(ctx, t.L(), db, replicationFactor, roachprod.AtLeastReplicationFactor)
 		require.NoError(t, err)

pkg/cmd/roachtest/tests/multi_store_remove.go

@@ -7,6 +7,7 @@ package tests
 
 import (
 	"context"
+	"math/rand"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
@@ -104,6 +105,21 @@ func runMultiStoreRemove(ctx context.Context, t test.Test, c cluster.Cluster) {
 		t.Fatal(err)
 	}
 
+	// Metamorphically enable the decommissioning nudger to get more test coverage
+	// on decommissioning nudger.
+	{
+		seed := timeutil.Now().UnixNano()
+		t.L().Printf("seed: %d", seed)
+		rng := rand.New(rand.NewSource(seed))
+
+		if rng.Intn(2) == 0 {
+			if _, err := conn.ExecContext(ctx, `SET CLUSTER SETTING kv.enqueue_in_replicate_queue_on_problem.interval = '10m'`); err != nil {
+				t.Fatal(err)
+			}
+			t.L().Printf("metamorphically enabled decommissioning nudger")
+		}
+	}
+
 	// Bring down node 1.
 	t.Status("removing store from n1")
 	node := c.Node(1)