kvserver: reproduce range-size backpressure for spanconfig updates

Spanconfig updates could be blocked by range-size backpressure when
range-size gets too large, creating a catch-22 situation where
garbage collection and protected timestamp removal operations were
prevented from running, which in turn prevented range
splitting and alleviation of the backpressure condition - leading to
cluster wide issues.

This commit adds a test that reproduces the issue by repeatedly writing
spanconfig updates for a single key until the range becomes too large
to split, triggering backpressure. The test demonstrates how spanconfig
updates (including protected timestamp deletions) get blocked by
backpressure, preventing the very operations needed to resolve
the backpressure condition.

Fixes: None
Release note: None

Author: dodeca12

pkg/kv/kvserver/client_replica_backpressure_test.go

@@ -8,27 +8,36 @@ package kvserver_test
 import (
 	"context"
 	"fmt"
+	math "math"
 	"net/url"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/spanconfig"
+	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/pgurlutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
+	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/errors"
+	"github.com/dustin/go-humanize"
 	"github.com/jackc/pgx/v5"
 	"github.com/stretchr/testify/require"
 )
@@ -332,3 +341,232 @@ func TestBackpressureNotAppliedWhenReducingRangeSize(t *testing.T) {
 		require.Error(t, <-upsertErrCh)
 	})
 }
+
+// TestSpanConfigUpdatesBlockedByRangeSizeBackpressureOnDefaultRanges
+// verifies that spanconfig updates are blocked by backpressure when the
+// `system.span_configurations` table range becomes full, recreating the issue.
+//
+// Test strategy:
+//  1. Configure `system.span_configurations` table range to be a small size (8 KiB).
+//  2. Write many large spanconfig records (2 KiB each) to fill up the range.
+//  3. Verify spanconfig updates fail due to backpressure when the range is full,
+//  4. This test recreates the scenario where spanconfig updates are blocked by
+//     backpressure.
+func TestSpanConfigUpdatesBlockedByRangeSizeBackpressureOnDefaultRanges(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	ctx := context.Background()
+
+	const (
+		overloadMaxRangeBytes = 8 << 10   // 8 KiB, a saner value than default 512 MiB for testing
+		overloadMinRangeBytes = 2 << 10   // 2 KiB
+		numWrites             = 16        // enough to hit backpressure for 8 KiB range & 2 KiB spanconfig
+		defaultMaxBytes       = 512 << 20 // default max bytes for a range
+	)
+	s := serverutils.StartServerOnly(t, base.TestServerArgs{})
+	defer s.Stopper().Stop(ctx)
+
+	store, err := s.GetStores().(*kvserver.Stores).GetStore(s.GetFirstStoreID())
+	require.NoError(t, err)
+
+	waitForSpanConfig := func(t *testing.T, tc serverutils.TestServerInterface,
+		tablePrefix roachpb.Key, expRangeMaxBytes int64) {
+		testutils.SucceedsSoon(t, func() error {
+			_, r := getFirstStoreReplica(t, tc, tablePrefix)
+			conf, err := r.LoadSpanConfig(ctx)
+			if err != nil {
+				return err
+			}
+			if conf.RangeMaxBytes != expRangeMaxBytes {
+				return fmt.Errorf("expected RangeMaxBytes %d, got %d",
+					expRangeMaxBytes, conf.RangeMaxBytes)
+			}
+			return nil
+		})
+	}
+
+	spanConfigTablePrefix := keys.SystemSQLCodec.TablePrefix(
+		keys.SpanConfigurationsTableID)
+
+	t.Logf("targeting span_configurations table at key: %s (table ID %d)\n",
+		spanConfigTablePrefix, keys.SpanConfigurationsTableID)
+
+	scratchKey, err := s.ScratchRange()
+	require.NoError(t, err)
+
+	testutils.SucceedsSoon(t, func() error {
+		repl := store.LookupReplica(roachpb.RKey(scratchKey))
+		if got := repl.GetMaxBytes(ctx); got != defaultMaxBytes {
+			return errors.Errorf(
+				"range max bytes values did not start at %d; got %d",
+				defaultMaxBytes, got)
+		}
+		return nil
+	})
+
+	systemSpanConfigurationsTableSpan := roachpb.Span{
+		Key:    spanConfigTablePrefix,
+		EndKey: spanConfigTablePrefix.PrefixEnd(),
+	}
+
+	target := spanconfig.MakeTargetFromSpan(systemSpanConfigurationsTableSpan)
+
+	systemSpanConfig := roachpb.SpanConfig{
+		RangeMaxBytes: overloadMaxRangeBytes,
+		RangeMinBytes: overloadMinRangeBytes,
+	}
+
+	configBytessdfsdf, err := protoutil.Marshal(&systemSpanConfig)
+	require.NoError(t, err)
+	t.Logf("marshalled systemSpanConfig size: %d bytes", len(configBytessdfsdf))
+
+	record, err := spanconfig.MakeRecord(target, systemSpanConfig)
+	require.NoError(t, err)
+
+	kvaccessor := s.SpanConfigKVAccessor().(spanconfig.KVAccessor)
+
+	err = kvaccessor.UpdateSpanConfigRecords(
+		ctx, []spanconfig.Target{target},
+		[]spanconfig.Record{record}, hlc.MinTimestamp, hlc.MaxTimestamp)
+	require.NoError(t, err)
+
+	waitForSpanConfig(t, s, spanConfigTablePrefix, overloadMaxRangeBytes)
+
+	// Check if the range is using our custom config.
+	repl := store.LookupReplica(keys.MustAddr(spanConfigTablePrefix))
+	if repl != nil {
+		conf, err := repl.LoadSpanConfig(ctx)
+		require.NoError(t, err)
+		t.Logf("current range config - RangeMaxBytes: %d bytes (%d MiB), "+
+			"RangeMinBytes: %d bytes (%d MiB)",
+			conf.RangeMaxBytes, humanize.Bytes(uint64(conf.RangeMaxBytes)),
+			conf.RangeMinBytes, humanize.Bytes(uint64(conf.RangeMinBytes)))
+
+	}
+
+	t.Logf("targeting span_configurations table at key: %s (table ID %d)\n",
+		spanConfigTablePrefix, keys.SpanConfigurationsTableID)
+
+	// Create a single target for the scratch range (this will be stored in system.span_configurations)
+	scratchTarget := spanconfig.MakeTargetFromSpan(roachpb.Span{
+		Key:    scratchKey,
+		EndKey: scratchKey.PrefixEnd(),
+	})
+
+	// This is a large spanconfig for a scratch range with relevant fields set
+	// to maximum int64 and int32 values. This is done to have a spanconfig that
+	// is large enough to trigger backpressure without having to write a million
+	// records.
+	// We want this config to be relatively large - this is done via setting
+	// values to have max values and multiple fields as this config gets
+	// marshalled into a protobuf and protobuf uses variant encoding, which
+	// means larger values take more bytes to encode.
+	spanConfig2KiB := roachpb.SpanConfig{ // 2078 bytes ~ 2 KiB.
+		RangeMaxBytes: math.MaxInt64,
+		RangeMinBytes: math.MaxInt64,
+		GCPolicy: roachpb.GCPolicy{
+			TTLSeconds: math.MaxInt32,
+			ProtectionPolicies: []roachpb.ProtectionPolicy{
+				{
+					ProtectedTimestamp: hlc.MaxTimestamp,
+				},
+				{
+					ProtectedTimestamp: hlc.MaxTimestamp,
+				},
+			},
+		},
+		NumReplicas: math.MaxInt32,
+		GlobalReads: true,
+		NumVoters:   math.MaxInt32,
+		VoterConstraints: []roachpb.ConstraintsConjunction{
+			{
+				Constraints: []roachpb.Constraint{
+					{Key: "max_key", Value: strings.Repeat("x", 1024)}, // very long constraint value
+				},
+			},
+		},
+		LeasePreferences: []roachpb.LeasePreference{
+			{
+				Constraints: []roachpb.Constraint{
+					{Key: "max_key", Value: strings.Repeat("y", 1024)}, // very long constraint value
+				},
+			},
+		},
+	}
+
+	configBytes, err := protoutil.Marshal(&spanConfig2KiB)
+	require.NoError(t, err)
+
+	require.GreaterOrEqual(t, len(configBytes), 2048,
+		"spanConfig2KiB should be at least 2 KiB in size")
+
+	// Create a record with the span configuration.
+	testRecord, err := spanconfig.MakeRecord(scratchTarget, spanConfig2KiB)
+	require.NoError(t, err)
+
+	// Write span configurations using KVAccessor.
+	// We expect this to fail due to backpressure.
+	var i int
+	for i = 0; i < numWrites; i++ {
+		// Use KVAccessor to update span configurations.
+		err = kvaccessor.UpdateSpanConfigRecords(ctx, nil,
+			[]spanconfig.Record{testRecord}, hlc.MinTimestamp, hlc.MaxTimestamp)
+		if err != nil {
+			break
+		}
+	}
+
+	// Assert that the operation failed due to backpressure.
+	require.Error(t, err,
+		"expected span config writes to fail due to backpressure, but they succeeded")
+
+	systemSpanConfigurationsTableSpanMVCCStats := roachpb.Span{
+		Key:    keys.SystemSQLCodec.TablePrefix(keys.SpanConfigurationsTableID),
+		EndKey: keys.SystemSQLCodec.TablePrefix(keys.SpanConfigurationsTableID + 1),
+	}
+
+	distSender := s.DistSenderI().(*kvcoord.DistSender)
+
+	// Track aggregate MVCC stats across all SpanConfigurationsTable ranges
+	var aggregateStats enginepb.MVCCStats
+	var rangeCount int
+
+	for key := systemSpanConfigurationsTableSpanMVCCStats.Key; key.Compare(systemSpanConfigurationsTableSpanMVCCStats.EndKey) < 0; {
+		desc, err := distSender.RangeDescriptorCache().Lookup(ctx, keys.MustAddr(key))
+		require.NoError(t, err)
+		d := desc.Desc
+
+		rangeRepl := store.LookupReplica(d.StartKey)
+		if rangeRepl != nil {
+			stats := rangeRepl.GetMVCCStats()
+			aggregateStats.Add(stats)
+			rangeCount++
+		}
+
+		// Move to next range.
+		key = d.EndKey.AsRawKey()
+		if key.Equal(roachpb.KeyMax) {
+			break
+		}
+	}
+
+	require.Greater(t, aggregateStats.Total(), int64(overloadMaxRangeBytes))
+
+	smallSpanConfig := roachpb.SpanConfig{
+		GCPolicy: roachpb.GCPolicy{
+			TTLSeconds: 0,
+		},
+	}
+
+	smallSpanconfigRecord, err := spanconfig.MakeRecord(scratchTarget, smallSpanConfig)
+	require.NoError(t, err)
+
+	smallSpanconfigRecordWriteErr := kvaccessor.UpdateSpanConfigRecords(ctx,
+		[]spanconfig.Target{scratchTarget}, []spanconfig.Record{smallSpanconfigRecord},
+		hlc.MinTimestamp, hlc.MaxTimestamp)
+
+	require.Error(t, smallSpanconfigRecordWriteErr,
+		"expected smallSpanconfigRecord write to fail due to backpressure")
+
+}