Merge #150172

150172: failureinjection/roachtest: add helper to spin up artificial MR cluster r=DarrylWong a=DarrylWong

The current network latency failure mode requires the user to specify latencies between every single node. However, one use case of this failure mode may to simulate a MR deployment, where specifying the latency pairings for every node can be burdensome.

This change adds some helpers to easily create an artificial MR cluster.

Release note: none
Informs: #138970

------

A while ago, it was [suggested that metamorphically simulating MR](https://cockroachlabs.slack.com/archives/C023S0V4YEB/p1739990528631999) clusters could provide more coverage. I was able to reproduce the bug found in #140967 with this approach by running the existing mixed version backup test with no other changes. I figured it might be interesting for `@Dev-Kyle` to try and get this working for the mixed version framework (time permitting).

I also did a comparison of our simulated MR cluster vs an actual geo distributed cluster and saw they both achieved roughly the same TPCC/YCSB performance.

Co-authored-by: DarrylWong <darryl@cockroachlabs.com>

Author: craig[bot]

pkg/cmd/roachtest/roachtestutil/utils.go

@@ -15,13 +15,15 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 	"github.com/cockroachdb/cockroach/pkg/roachprod"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/config"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/failureinjection/failures"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
 	"github.com/cockroachdb/cockroach/pkg/util"
@@ -284,3 +286,53 @@ func PrefixCmdOutputWithTimestamp(cmd string) string {
 	awkCmd := `awk 'NF { cmd="date +\"%H:%M:%S\""; cmd | getline ts; close(cmd); print ts ":", $0; next } { print }'`
 	return fmt.Sprintf(`bash -c '%s' 2>&1 |`, cmd) + awkCmd
 }
+
+// SimulateMultiRegionCluster injects artificial latency between nodes
+// to simulate a multi-region cluster with nodes distributed according
+// to regionToNodeMap. The actual cluster itself must have been provisioned
+// in the same region/zone.
+func SimulateMultiRegionCluster(
+	ctx context.Context,
+	t test.Test,
+	c cluster.Cluster,
+	regionToNodeMap failures.RegionToNodes,
+	l *logger.Logger,
+) (func(), error) {
+	latencyFailer, err := c.GetFailer(l, c.All(), failures.NetworkLatencyName)
+	if err != nil {
+		return nil, err
+	}
+
+	args, err := failures.MakeNetworkLatencyArgs(regionToNodeMap)
+	if err != nil {
+		return nil, err
+	}
+
+	failerLogger, fileName, err := LoggerForCmd(l, c.All(), "MultiRegionClusterSetup")
+	if err != nil {
+		return nil, err
+	}
+	l.Printf("Simulating Multi Region cluster; details in %s.log", fileName)
+
+	if err = latencyFailer.Setup(ctx, failerLogger, args); err != nil {
+		return nil, err
+	}
+
+	// We want to inject the failure mode after service registration but before
+	// the cluster actually starts. We also want to only inject it the first time,
+	// as it will persist through restarts.
+	var once sync.Once
+	c.RegisterClusterHook("inject multi-region latency", option.PreStartHook, time.Minute, func(ctx context.Context) error {
+		once.Do(func() {
+			err = latencyFailer.Inject(ctx, failerLogger, args)
+		})
+		return err
+	})
+	cleanupFunc := func() {
+		if err = latencyFailer.Cleanup(context.Background(), failerLogger); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	return cleanupFunc, nil
+}

pkg/cmd/roachtest/tests/tpcc.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/task"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/failureinjection/failures"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/prometheus"
@@ -920,6 +921,43 @@ func registerTPCC(r registry.Registry) {
 		},
 	})
 
+	// This test runs TPCC on a simulated multi-region cluster by injecting network latency
+	// failure modes. This tests that our artificial latency failure mode used to simulate MR works,
+	// as well as offers a cheaper way to test MR TPCC without spinning up a distributed cluster.
+	r.Add(registry.TestSpec{
+		Name:              fmt.Sprintf("tpcc/headroom/%s/artificial-multi-region", headroomSpec.String()),
+		Owner:             registry.OwnerTestEng,
+		Benchmark:         true,
+		CompatibleClouds:  registry.AllClouds,
+		Suites:            registry.ManualOnly,
+		Cluster:           headroomSpec,
+		Timeout:           4 * time.Hour,
+		EncryptionSupport: registry.EncryptionMetamorphic,
+		Leases:            registry.MetamorphicLeases,
+		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
+			// Note that the workload node is placed in us-east as it's in the middle.
+			regionToNodes := failures.RegionToNodes{
+				failures.USEast:     {1, 4},
+				failures.USWest:     {2},
+				failures.EuropeWest: {3},
+			}
+
+			cleanup, err := roachtestutil.SimulateMultiRegionCluster(ctx, t, c, regionToNodes, t.L())
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			maxWarehouses := maxSupportedTPCCWarehouses(*t.BuildVersion(), c.Cloud(), c.Spec())
+			headroomWarehouses := int(float64(maxWarehouses) * 0.7)
+			t.L().Printf("computed headroom warehouses of %d\n", headroomWarehouses)
+			runTPCC(ctx, t, t.L(), c, tpccOptions{
+				Warehouses: headroomWarehouses,
+				Duration:   120 * time.Minute,
+				SetupType:  usingImport,
+			})
+		},
+	})
+
 	// Setup multi-region tests.
 	{
 		mrSetup := []struct {

pkg/roachprod/failureinjection/failures/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
         "failer.go",
         "failure.go",
         "latency.go",
+        "multiregion_latency.go",
         "network_partition.go",
         "noop.go",
         "option.go",

pkg/roachprod/failureinjection/failures/latency.go

@@ -39,8 +39,11 @@ type ArtificialLatency struct {
 	Delay       time.Duration
 }
 
-func (l *ArtificialLatency) String() string {
-	return fmt.Sprintf("%d-%d-%s", l.Source, l.Destination, l.Delay)
+// FilterName returns the unique name for each filter. Note that the filter name maps
+// many source nodes to single destination node. Each node to node latency needs its
+// own filter, but we can inject the same rule on multiple source nodes in one shot.
+func (l *ArtificialLatency) FilterName(destNode install.Node) string {
+	return fmt.Sprintf("%d-%d-%s", l.Source, destNode, l.Delay)
 }
 
 type NetworkLatencyArgs struct {
@@ -179,10 +182,10 @@ func (f *NetworkLatency) Inject(ctx context.Context, l *logger.Logger, args Fail
 			// Enforce we don't have duplicate rules, as it complicates the removal process of filters
 			// and is something the user likely didn't intend.
 			class := f.findNextOpenClass()
-			if _, ok := f.filterNameToClassMap[latency.String()]; ok {
+			if _, ok := f.filterNameToClassMap[latency.FilterName(dest)]; ok {
 				return errors.Newf("failed trying to inject ArtificialLatency, rule already exists: %+v", latency)
 			}
-			f.filterNameToClassMap[latency.String()] = class
+			f.filterNameToClassMap[latency.FilterName(dest)] = class
 			handle := 10 * class
 
 			cmd := failScriptEarlyCmd
@@ -210,7 +213,7 @@ func (f *NetworkLatency) Recover(ctx context.Context, l *logger.Logger, args Fai
 				return err
 			}
 
-			class, ok := f.filterNameToClassMap[latency.String()]
+			class, ok := f.filterNameToClassMap[latency.FilterName(dest)]
 			if !ok {
 				return errors.New("failed trying to recover latency failure, ArtificialLatency rule was not found: %+v")
 			}
@@ -229,7 +232,7 @@ func (f *NetworkLatency) Recover(ctx context.Context, l *logger.Logger, args Fai
 			if class < f.nextAvailableClass {
 				f.nextAvailableClass = class
 			}
-			delete(f.filterNameToClassMap, latency.String())
+			delete(f.filterNameToClassMap, latency.FilterName(dest))
 		}
 	}
 	return nil

pkg/roachprod/failureinjection/failures/multiregion_latency.go

@@ -0,0 +1,117 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package failures
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
+)
+
+type (
+	// Region represents a region in which a node can be located.
+	Region string
+	// RegionToNodes is a convenience type that maps regions to their nodes.
+	RegionToNodes map[Region][]install.Node
+	// LatencyMap is a mapping of all the one way latencies between regions,
+	// i.e. 0.5 * RTT.
+	LatencyMap map[Region]map[Region]time.Duration
+	// roundTripLatency represents the RTT from RegionA to RegionB.
+	roundTripLatency struct {
+		RegionA Region
+		RegionB Region
+		Latency time.Duration
+	}
+)
+
+const (
+	USEast     Region = "us-east"
+	USWest     Region = "us-west"
+	EuropeWest Region = "europe-west"
+)
+
+// createLatencyMap creates a LatencyMap from a slice of roundTripLatency.
+// N.B. Latencies are assumed to be symmetric. The latency from region A
+// to region B is the same as the latency from region B to region A, i.e.
+// 0.5 * RTT.
+func createLatencyMap(roundTripLatency []roundTripLatency) LatencyMap {
+	latencyMap := make(map[Region]map[Region]time.Duration)
+	for _, rtt := range roundTripLatency {
+		if _, ok := latencyMap[rtt.RegionA]; !ok {
+			latencyMap[rtt.RegionA] = make(map[Region]time.Duration)
+		}
+		if _, ok := latencyMap[rtt.RegionB]; !ok {
+			latencyMap[rtt.RegionB] = make(map[Region]time.Duration)
+		}
+		latencyMap[rtt.RegionA][rtt.RegionB] = rtt.Latency / 2
+		latencyMap[rtt.RegionB][rtt.RegionA] = rtt.Latency / 2
+	}
+	return latencyMap
+}
+
+// defaultLatencyMap returns a LatencyMap with estimated latencies between
+// VMs in different regions. Numbers are copied from the GCP console at
+// of the time of this comment:
+// https://console.cloud.google.com/net-intelligence/performance/global-dashboard
+//
+// N.B. Assumes the actual roachprod cluster was created in the same region/zone, i.e.
+// the original latency is negligible (gce claims sub 1ms for intra zone).
+var defaultLatencyMap = func() LatencyMap {
+	regionLatencies := []roundTripLatency{
+		{
+			RegionA: USEast,
+			RegionB: USWest,
+			Latency: 64 * time.Millisecond,
+		},
+		{
+			RegionA: USEast,
+			RegionB: EuropeWest,
+			Latency: 86 * time.Millisecond,
+		},
+		{
+			RegionA: USWest,
+			RegionB: EuropeWest,
+			Latency: 132 * time.Millisecond,
+		},
+	}
+	return createLatencyMap(regionLatencies)
+}()
+
+// MakeNetworkLatencyArgs returns the NetworkLatencyArgs to simulate the latency
+// of a multiregion cluster with the provided mapping.
+func MakeNetworkLatencyArgs(regionToNodeMap RegionToNodes) (NetworkLatencyArgs, error) {
+	artificialLatencies := make([]ArtificialLatency, 0)
+	LatencyMappingNotFoundErr := func(regionA, regionB Region) error {
+		return fmt.Errorf("no latency mapping found from region %s to %s", regionA, regionB)
+	}
+	for regionA, srcNodes := range regionToNodeMap {
+		for regionB, destNodes := range regionToNodeMap {
+			if regionA == regionB {
+				continue
+			}
+			if defaultLatencyMap[regionA] == nil {
+				return NetworkLatencyArgs{}, LatencyMappingNotFoundErr(regionA, regionB)
+			}
+			delay, ok := defaultLatencyMap[regionA][regionB]
+			if !ok {
+				return NetworkLatencyArgs{}, LatencyMappingNotFoundErr(regionA, regionB)
+			}
+
+			artificialLatencies = append(artificialLatencies, ArtificialLatency{
+				Source:      srcNodes,
+				Destination: destNodes,
+				Delay:       delay,
+			})
+		}
+	}
+
+	args := NetworkLatencyArgs{
+		ArtificialLatencies: artificialLatencies,
+	}
+
+	return args, nil
+}