Merge #150564 #153145 #153326

150564: ui: rework network page r=tbg a=tbg

Trying to make it a bit more functional. First, we put bytes sent/received first. We drop packets sent/received, which largely duplicates this (I could be talked out of that).

Packet errors and drops are jammed into a single chart. Step changes are (hopefully) still visible, and it avoids having to scroll through a set of empty graphs on the way down in the common case of no issues. (I might be wrong about this, but I've never seen errors, for example, only drops).

I added the new TCP retransmits chart from #149928.

The proxy charts probably not useful most of the time, but I kept them around. _If_ they do something, I assume we want to know.

| Old Chart Layout                      | New Chart Layout                         |
| :---------------------------------------- | :--------------------------------------- |
| Network Bytes Received                    | Network Bytes Sent                       |
| Network Packets Received                  | Network Bytes Received                   |
| Network Packet Errors on Receive          | RPC Heartbeat Latency: 50th percentile   |
| Network Packet Drops on Receive           | RPC Heartbeat Latency: 99th percentile   |
| Network Bytes Sent                        | Unhealthy RPC Connections                |
| Network Packets Sent                      | Network Packet Errors and Drops          |
| Network Packet Errors on Send             | TCP Retransmits                          |
| Network Packet Drops on Send              | Proxy requests                           |
| RPC Heartbeat Latency: 50th percentile    | Proxy request errors                     |
| RPC Heartbeat Latency: 99th percentile    | Proxy forwards                           |
| Unhealthy RPC Connections                 | Proxy forward errors                     |
| Proxy requests                            |                                          |
| Proxy request errors                      |                                          |
| Proxy forwards                            |                                          |

Epic: none

153145: kvserver: exec-trace TestReplicateQueueRebalanceMultiStore r=tbg a=tbg

It failed in a way that looked like network latency, but there is no non-local networking or artificial delay in this test. Enable execution tracing this
test to maybe understand this better next time.

Closes #153137.

Epic: none

153326: allocatorimpl: rm TestAllocatorFullDisks r=tbg a=tbg

It's been flaky forever, and we have asim coverage for full disks (asim/tests/testdata/non_rand/example_fulldisk[^1])

[^1]: https://github.com/cockroachdb/cockroach/blob/8b1e1a9e399ca84a87c686d0106279e0c5904871/pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk.txt#L5

This and its backports will:

Close #153225.
Close #152897.
Close #152317.
Close #152056
Close #150456.

Epic: none

Co-authored-by: Tobias Grieger <tobias.b.grieger@gmail.com>

Author: craig[bot]

pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go

@@ -8785,160 +8785,6 @@ func (ts *testStore) rebalance(ots *testStore, bytes int64, qps float64, do Disk
 	ots.Capacity.LogicalBytes += bytes
 }
 
-func (ts *testStore) compact() {
-	ts.Capacity.Used = ts.Capacity.LogicalBytes
-	ts.Capacity.Available = ts.Capacity.Capacity - ts.Capacity.Used
-}
-
-func TestAllocatorFullDisks(t *testing.T) {
-	defer leaktest.AfterTest(t)()
-	defer log.Scope(t).Close(t)
-
-	ctx := context.Background()
-	stopper := stop.NewStopper()
-	defer stopper.Stop(ctx)
-
-	st := cluster.MakeTestingClusterSettings()
-	tr := tracing.NewTracer()
-	clock := hlc.NewClockForTesting(nil)
-
-	g := gossip.NewTest(1, stopper, metric.NewRegistry())
-
-	liveness.TimeUntilNodeDead.Override(ctx, &st.SV, liveness.TestTimeUntilNodeDeadOff)
-
-	const generations = 100
-	const nodes = 20
-	const capacity = (1 << 30) + 1
-	const rangeSize = 16 << 20
-
-	mockNodeLiveness := storepool.NewMockNodeLiveness(livenesspb.NodeLivenessStatus_LIVE)
-	sp := storepool.NewStorePool(
-		log.MakeTestingAmbientContext(tr),
-		st,
-		g,
-		clock,
-		func() int {
-			return nodes
-		},
-		mockNodeLiveness.NodeLivenessFunc,
-		false, /* deterministic */
-	)
-	alloc := MakeAllocator(st, false /* deterministic */, func(id roachpb.NodeID) (time.Duration, bool) {
-		return 0, false
-	}, nil)
-
-	var wg sync.WaitGroup
-	g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStoreDescPrefix),
-		func(_ string, _ roachpb.Value, _ int64) { wg.Done() },
-		// Redundant callbacks are required by this test.
-		gossip.Redundant)
-
-	do := makeDiskCapacityOptions(&st.SV)
-
-	// Each range is equally sized (16mb), we want the number of ranges per node,
-	// when their size is added, to be no greater than the full disk rebalance
-	// threshold (0.925%) e.g for below:
-	//   capacity  = 1024mb
-	//   rangeSize = 16mb
-	//   threshold = 0.925
-	//   rangesPerNode =  ⌊1024mb * 0.925 / 16mb⌋ = 59
-	rangesPerNode := int(math.Floor(capacity * do.RebalanceToThreshold / rangeSize))
-	rangesToAdd := rangesPerNode * nodes
-
-	// Initialize testStores.
-	var testStores [nodes]testStore
-	for i := 0; i < len(testStores); i++ {
-		// Don't immediately reclaim disk space from removed ranges. This mimics
-		// range deletions don't immediately reclaim disk space in rocksdb.
-		testStores[i].immediateCompaction = false
-		testStores[i].StoreID = roachpb.StoreID(i)
-		testStores[i].Node = roachpb.NodeDescriptor{NodeID: roachpb.NodeID(i)}
-		testStores[i].Capacity = roachpb.StoreCapacity{Capacity: capacity, Available: capacity}
-	}
-	// Initialize the cluster with a single range.
-	testStores[0].add(rangeSize, 0)
-	rangesAdded := 1
-
-	for i := 0; i < generations; i++ {
-		// First loop through test stores and randomly add data.
-		for j := 0; j < len(testStores); j++ {
-			if mockNodeLiveness.NodeLivenessFunc(roachpb.NodeID(j)) == livenesspb.NodeLivenessStatus_DEAD {
-				continue
-			}
-			ts := &testStores[j]
-			// Add [0,2) ranges to the node, simulating splits and data growth.
-			toAdd := alloc.randGen.Intn(2)
-			for k := 0; k < toAdd; k++ {
-				if rangesAdded < rangesToAdd {
-					ts.add(rangeSize, 0)
-					rangesAdded++
-				}
-			}
-			if ts.Capacity.Available <= 0 {
-				t.Errorf("testStore %d ran out of space during generation %d (rangesAdded=%d/%d): %+v",
-					j, i, rangesAdded, rangesToAdd, ts.Capacity)
-				mockNodeLiveness.SetNodeStatus(roachpb.NodeID(j), livenesspb.NodeLivenessStatus_DEAD)
-			}
-			wg.Add(1)
-			if err := g.AddInfoProto(gossip.MakeStoreDescKey(roachpb.StoreID(j)), &ts.StoreDescriptor, 0); err != nil {
-				t.Fatal(err)
-			}
-		}
-		wg.Wait()
-
-		// Loop through each store a number of times and maybe rebalance.
-		for j := 0; j < 10; j++ {
-			for k := 0; k < len(testStores); k++ {
-				if mockNodeLiveness.NodeLivenessFunc(roachpb.NodeID(k)) == livenesspb.NodeLivenessStatus_DEAD {
-					continue
-				}
-				ts := &testStores[k]
-				// Rebalance until there's no more rebalancing to do.
-				if ts.Capacity.RangeCount > 0 {
-					var rangeUsageInfo allocator.RangeUsageInfo
-					target, _, details, ok := alloc.RebalanceVoter(
-						ctx,
-						sp,
-						emptySpanConfig(),
-						nil,
-						[]roachpb.ReplicaDescriptor{{NodeID: ts.Node.NodeID, StoreID: ts.StoreID}},
-						nil,
-						rangeUsageInfo,
-						storepool.StoreFilterThrottled,
-						alloc.ScorerOptions(ctx),
-					)
-					if ok {
-						if log.V(1) {
-							log.Dev.Infof(ctx, "rebalancing to %v; details: %s", target, details)
-						}
-						testStores[k].rebalance(&testStores[int(target.StoreID)], rangeSize, 0 /* qps */, do)
-					}
-				}
-				// Gossip occasionally, as real Stores do when replicas move around.
-				if j%3 == 2 {
-					wg.Add(1)
-					if err := g.AddInfoProto(gossip.MakeStoreDescKey(roachpb.StoreID(j)), &ts.StoreDescriptor, 0); err != nil {
-						t.Fatal(err)
-					}
-				}
-			}
-		}
-
-		// Simulate rocksdb compactions freeing up disk space.
-		for j := 0; j < len(testStores); j++ {
-			if mockNodeLiveness.NodeLivenessFunc(roachpb.NodeID(j)) != livenesspb.NodeLivenessStatus_DEAD {
-				ts := &testStores[j]
-				if ts.Capacity.Available <= 0 {
-					t.Errorf("testStore %d ran out of space during generation %d: %+v", j, i, ts.Capacity)
-					mockNodeLiveness.SetNodeStatus(roachpb.NodeID(j), livenesspb.NodeLivenessStatus_DEAD)
-				} else {
-					ts.compact()
-				}
-			}
-		}
-	}
-}
-
 func Example_rangeCountRebalancing() {
 	testStores := make([]testStore, 20)
 	rebalanceFn := func(ctx context.Context, ts *testStore, testStores []testStore, alloc *Allocator, storePool *storepool.StorePool) {

pkg/kv/kvserver/replicate_queue_test.go

@@ -176,10 +176,15 @@ func TestReplicateQueueRebalance(t *testing.T) {
 // rebalances the replicas and leases.
 func TestReplicateQueueRebalanceMultiStore(t *testing.T) {
 	defer leaktest.AfterTest(t)()
-	defer log.Scope(t).Close(t)
-	skip.UnderRace(t)
-	skip.UnderShort(t)
-	skip.UnderDeadlock(t)
+	skip.UnderDuress(t) // eight stores is too much under duress
+	scope := log.Scope(t)
+	defer scope.Close(t)
+
+	// The test exhibited an interesting failure mode that we want
+	// to be able to better investigate should it reoccur.
+	// See: https://github.com/cockroachdb/cockroach/issues/153137
+	// and https://cockroachlabs.slack.com/archives/G01G8LK77DK/p1757330830964639.
+	defer testutils.StartExecTrace(t, scope.GetDirectory()).Finish(t)
 
 	testCases := []struct {
 		name          string