Skip to content

Commit 1c5a706

Browse files
craig[bot]asg0451
craig[bot]
and
asg0451
committed
Merge #154545
154545: changefeedccl: clean up blocking buffer metrics r=andyyang890,log-head a=asg0451 This commit removes the blocking buffer metrics that were not broken down by buffer type (rangefeed and aggregator), with the exception of the memory accounting metrics, which have had the broken down versions removed because the breakdown was incorrect in their case. Fixes #147624 Removed because the buffer type breakdown was inappropriate: - changefeed.buffer_entries.allocated_mem.aggregator - changefeed.buffer_entries.allocated_mem.rangefeed - changefeed.buffer_entries_mem.acquired.aggregator - changefeed.buffer_entries_mem.acquired.rangefeed - changefeed.buffer_entries_mem.released.aggregator - changefeed.buffer_entries_mem.released.rangefeed Removed because the buffer type breakdown (eg changefeed.buffer_entries.flush.rangefeed) is sufficient: - changefeed.buffer_entries.flush - changefeed.buffer_entries.in - changefeed.buffer_entries.out - changefeed.buffer_entries.kv - changefeed.buffer_entries.released - changefeed.buffer_pushback_nanos Release note (ops change): Cleaned up several redundant and misleading metrics. Co-authored-by: Miles Frankel <miles.frankel@cockroachlabs.com>
2 parents d1faa08 + 1155d58 commit 1c5a706

File tree

10 files changed

+121
-266
lines changed

10 files changed

+121
-266
lines changed

‎docs/generated/metrics/metrics.yaml‎

Lines changed: 14 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -1321,32 +1321,9 @@ layers:
13211321
unit: BYTES
13221322
aggregation: AVG
13231323
derivative: NONE
1324-
- name: changefeed.buffer_entries.allocated_mem.aggregator
1325-
exported_name: changefeed_buffer_entries_allocated_mem_aggregator
1326-
description: Current quota pool memory allocation - between the kvfeed and the sink
1327-
y_axis_label: Bytes
1328-
type: GAUGE
1329-
unit: BYTES
1330-
aggregation: AVG
1331-
derivative: NONE
1332-
- name: changefeed.buffer_entries.allocated_mem.rangefeed
1333-
exported_name: changefeed_buffer_entries_allocated_mem_rangefeed
1334-
description: Current quota pool memory allocation - between the rangefeed and the kvfeed
1335-
y_axis_label: Bytes
1336-
type: GAUGE
1337-
unit: BYTES
1338-
aggregation: AVG
1339-
derivative: NONE
1340-
- name: changefeed.buffer_entries.flush
1341-
exported_name: changefeed_buffer_entries_flush
1342-
description: Number of flush elements added to the buffer
1343-
y_axis_label: Events
1344-
type: COUNTER
1345-
unit: COUNT
1346-
aggregation: AVG
1347-
derivative: NON_NEGATIVE_DERIVATIVE
13481324
- name: changefeed.buffer_entries.flush.aggregator
13491325
exported_name: changefeed_buffer_entries_flush_aggregator
1326+
labeled_name: 'changefeed.buffer_entries.flush{buffer_type: aggregator}'
13501327
description: Number of flush elements added to the buffer - between the kvfeed and the sink
13511328
y_axis_label: Events
13521329
type: COUNTER
@@ -1355,22 +1332,16 @@ layers:
13551332
derivative: NON_NEGATIVE_DERIVATIVE
13561333
- name: changefeed.buffer_entries.flush.rangefeed
13571334
exported_name: changefeed_buffer_entries_flush_rangefeed
1335+
labeled_name: 'changefeed.buffer_entries.flush{buffer_type: rangefeed}'
13581336
description: Number of flush elements added to the buffer - between the rangefeed and the kvfeed
13591337
y_axis_label: Events
13601338
type: COUNTER
13611339
unit: COUNT
13621340
aggregation: AVG
13631341
derivative: NON_NEGATIVE_DERIVATIVE
1364-
- name: changefeed.buffer_entries.in
1365-
exported_name: changefeed_buffer_entries_in
1366-
description: Total entries entering the buffer between raft and changefeed sinks
1367-
y_axis_label: Entries
1368-
type: COUNTER
1369-
unit: COUNT
1370-
aggregation: AVG
1371-
derivative: NON_NEGATIVE_DERIVATIVE
13721342
- name: changefeed.buffer_entries.in.aggregator
13731343
exported_name: changefeed_buffer_entries_in_aggregator
1344+
labeled_name: 'changefeed.buffer_entries.in{buffer_type: aggregator}'
13741345
description: Total entries entering the buffer between raft and changefeed sinks - between the kvfeed and the sink
13751346
y_axis_label: Entries
13761347
type: COUNTER
@@ -1379,22 +1350,16 @@ layers:
13791350
derivative: NON_NEGATIVE_DERIVATIVE
13801351
- name: changefeed.buffer_entries.in.rangefeed
13811352
exported_name: changefeed_buffer_entries_in_rangefeed
1353+
labeled_name: 'changefeed.buffer_entries.in{buffer_type: rangefeed}'
13821354
description: Total entries entering the buffer between raft and changefeed sinks - between the rangefeed and the kvfeed
13831355
y_axis_label: Entries
13841356
type: COUNTER
13851357
unit: COUNT
13861358
aggregation: AVG
13871359
derivative: NON_NEGATIVE_DERIVATIVE
1388-
- name: changefeed.buffer_entries.kv
1389-
exported_name: changefeed_buffer_entries_kv
1390-
description: Number of kv elements added to the buffer
1391-
y_axis_label: Events
1392-
type: COUNTER
1393-
unit: COUNT
1394-
aggregation: AVG
1395-
derivative: NON_NEGATIVE_DERIVATIVE
13961360
- name: changefeed.buffer_entries.kv.aggregator
13971361
exported_name: changefeed_buffer_entries_kv_aggregator
1362+
labeled_name: 'changefeed.buffer_entries.kv{buffer_type: aggregator}'
13981363
description: Number of kv elements added to the buffer - between the kvfeed and the sink
13991364
y_axis_label: Events
14001365
type: COUNTER
@@ -1403,22 +1368,16 @@ layers:
14031368
derivative: NON_NEGATIVE_DERIVATIVE
14041369
- name: changefeed.buffer_entries.kv.rangefeed
14051370
exported_name: changefeed_buffer_entries_kv_rangefeed
1371+
labeled_name: 'changefeed.buffer_entries.kv{buffer_type: rangefeed}'
14061372
description: Number of kv elements added to the buffer - between the rangefeed and the kvfeed
14071373
y_axis_label: Events
14081374
type: COUNTER
14091375
unit: COUNT
14101376
aggregation: AVG
14111377
derivative: NON_NEGATIVE_DERIVATIVE
1412-
- name: changefeed.buffer_entries.out
1413-
exported_name: changefeed_buffer_entries_out
1414-
description: Total entries leaving the buffer between raft and changefeed sinks
1415-
y_axis_label: Entries
1416-
type: COUNTER
1417-
unit: COUNT
1418-
aggregation: AVG
1419-
derivative: NON_NEGATIVE_DERIVATIVE
14201378
- name: changefeed.buffer_entries.out.aggregator
14211379
exported_name: changefeed_buffer_entries_out_aggregator
1380+
labeled_name: 'changefeed.buffer_entries.out{buffer_type: aggregator}'
14221381
description: Total entries leaving the buffer between raft and changefeed sinks - between the kvfeed and the sink
14231382
y_axis_label: Entries
14241383
type: COUNTER
@@ -1427,22 +1386,16 @@ layers:
14271386
derivative: NON_NEGATIVE_DERIVATIVE
14281387
- name: changefeed.buffer_entries.out.rangefeed
14291388
exported_name: changefeed_buffer_entries_out_rangefeed
1389+
labeled_name: 'changefeed.buffer_entries.out{buffer_type: rangefeed}'
14301390
description: Total entries leaving the buffer between raft and changefeed sinks - between the rangefeed and the kvfeed
14311391
y_axis_label: Entries
14321392
type: COUNTER
14331393
unit: COUNT
14341394
aggregation: AVG
14351395
derivative: NON_NEGATIVE_DERIVATIVE
1436-
- name: changefeed.buffer_entries.released
1437-
exported_name: changefeed_buffer_entries_released
1438-
description: Total entries processed, emitted and acknowledged by the sinks
1439-
y_axis_label: Entries
1440-
type: COUNTER
1441-
unit: COUNT
1442-
aggregation: AVG
1443-
derivative: NON_NEGATIVE_DERIVATIVE
14441396
- name: changefeed.buffer_entries.released.aggregator
14451397
exported_name: changefeed_buffer_entries_released_aggregator
1398+
labeled_name: 'changefeed.buffer_entries.released{buffer_type: aggregator}'
14461399
description: Total entries processed, emitted and acknowledged by the sinks - between the kvfeed and the sink
14471400
y_axis_label: Entries
14481401
type: COUNTER
@@ -1451,22 +1404,16 @@ layers:
14511404
derivative: NON_NEGATIVE_DERIVATIVE
14521405
- name: changefeed.buffer_entries.released.rangefeed
14531406
exported_name: changefeed_buffer_entries_released_rangefeed
1407+
labeled_name: 'changefeed.buffer_entries.released{buffer_type: rangefeed}'
14541408
description: Total entries processed, emitted and acknowledged by the sinks - between the rangefeed and the kvfeed
14551409
y_axis_label: Entries
14561410
type: COUNTER
14571411
unit: COUNT
14581412
aggregation: AVG
14591413
derivative: NON_NEGATIVE_DERIVATIVE
1460-
- name: changefeed.buffer_entries.resolved
1461-
exported_name: changefeed_buffer_entries_resolved
1462-
description: Number of resolved elements added to the buffer
1463-
y_axis_label: Events
1464-
type: COUNTER
1465-
unit: COUNT
1466-
aggregation: AVG
1467-
derivative: NON_NEGATIVE_DERIVATIVE
14681414
- name: changefeed.buffer_entries.resolved.aggregator
14691415
exported_name: changefeed_buffer_entries_resolved_aggregator
1416+
labeled_name: 'changefeed.buffer_entries.resolved{buffer_type: aggregator}'
14701417
description: Number of resolved elements added to the buffer - between the kvfeed and the sink
14711418
y_axis_label: Events
14721419
type: COUNTER
@@ -1475,6 +1422,7 @@ layers:
14751422
derivative: NON_NEGATIVE_DERIVATIVE
14761423
- name: changefeed.buffer_entries.resolved.rangefeed
14771424
exported_name: changefeed_buffer_entries_resolved_rangefeed
1425+
labeled_name: 'changefeed.buffer_entries.resolved{buffer_type: rangefeed}'
14781426
description: Number of resolved elements added to the buffer - between the rangefeed and the kvfeed
14791427
y_axis_label: Events
14801428
type: COUNTER
@@ -1489,22 +1437,6 @@ layers:
14891437
unit: COUNT
14901438
aggregation: AVG
14911439
derivative: NON_NEGATIVE_DERIVATIVE
1492-
- name: changefeed.buffer_entries_mem.acquired.aggregator
1493-
exported_name: changefeed_buffer_entries_mem_acquired_aggregator
1494-
description: Total amount of memory acquired for entries as they enter the system - between the kvfeed and the sink
1495-
y_axis_label: Entries
1496-
type: COUNTER
1497-
unit: COUNT
1498-
aggregation: AVG
1499-
derivative: NON_NEGATIVE_DERIVATIVE
1500-
- name: changefeed.buffer_entries_mem.acquired.rangefeed
1501-
exported_name: changefeed_buffer_entries_mem_acquired_rangefeed
1502-
description: Total amount of memory acquired for entries as they enter the system - between the rangefeed and the kvfeed
1503-
y_axis_label: Entries
1504-
type: COUNTER
1505-
unit: COUNT
1506-
aggregation: AVG
1507-
derivative: NON_NEGATIVE_DERIVATIVE
15081440
- name: changefeed.buffer_entries_mem.released
15091441
exported_name: changefeed_buffer_entries_mem_released
15101442
description: Total amount of memory released by the entries after they have been emitted
@@ -1513,32 +1445,9 @@ layers:
15131445
unit: COUNT
15141446
aggregation: AVG
15151447
derivative: NON_NEGATIVE_DERIVATIVE
1516-
- name: changefeed.buffer_entries_mem.released.aggregator
1517-
exported_name: changefeed_buffer_entries_mem_released_aggregator
1518-
description: Total amount of memory released by the entries after they have been emitted - between the kvfeed and the sink
1519-
y_axis_label: Entries
1520-
type: COUNTER
1521-
unit: COUNT
1522-
aggregation: AVG
1523-
derivative: NON_NEGATIVE_DERIVATIVE
1524-
- name: changefeed.buffer_entries_mem.released.rangefeed
1525-
exported_name: changefeed_buffer_entries_mem_released_rangefeed
1526-
description: Total amount of memory released by the entries after they have been emitted - between the rangefeed and the kvfeed
1527-
y_axis_label: Entries
1528-
type: COUNTER
1529-
unit: COUNT
1530-
aggregation: AVG
1531-
derivative: NON_NEGATIVE_DERIVATIVE
1532-
- name: changefeed.buffer_pushback_nanos
1533-
exported_name: changefeed_buffer_pushback_nanos
1534-
description: Total time spent waiting while the buffer was full
1535-
y_axis_label: Nanoseconds
1536-
type: COUNTER
1537-
unit: NANOSECONDS
1538-
aggregation: AVG
1539-
derivative: NON_NEGATIVE_DERIVATIVE
15401448
- name: changefeed.buffer_pushback_nanos.aggregator
15411449
exported_name: changefeed_buffer_pushback_nanos_aggregator
1450+
labeled_name: 'changefeed.buffer_pushback_nanos{buffer_type: aggregator}'
15421451
description: Total time spent waiting while the buffer was full - between the kvfeed and the sink
15431452
y_axis_label: Nanoseconds
15441453
type: COUNTER
@@ -1547,6 +1456,7 @@ layers:
15471456
derivative: NON_NEGATIVE_DERIVATIVE
15481457
- name: changefeed.buffer_pushback_nanos.rangefeed
15491458
exported_name: changefeed_buffer_pushback_nanos_rangefeed
1459+
labeled_name: 'changefeed.buffer_pushback_nanos{buffer_type: rangefeed}'
15501460
description: Total time spent waiting while the buffer was full - between the rangefeed and the kvfeed
15511461
y_axis_label: Nanoseconds
15521462
type: COUNTER

‎pkg/ccl/changefeedccl/changefeed_processors.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ func (ca *changeAggregator) startKVFeed(
507507
}
508508
buf := kvevent.NewThrottlingBuffer(
509509
kvevent.NewMemBuffer(kvFeedMemMon.MakeBoundAccount(), &cfg.Settings.SV,
510-
&ca.metrics.KVFeedMetrics.AggregatorBufferMetricsWithCompat, options...),
510+
&ca.metrics.KVFeedMetrics.AggregatorBufferMetrics, options...),
511511
cdcutils.NodeLevelThrottler(&cfg.Settings.SV, &ca.metrics.ThrottleMetrics))
512512

513513
// KVFeed takes ownership of the kvevent.Writer portion of the buffer, while

‎pkg/ccl/changefeedccl/changefeed_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6558,10 +6558,10 @@ func TestChangefeedMonitoring(t *testing.T) {
65586558
if c := s.Server.MustGetSQLCounter(`changefeed.max_behind_nanos`); c != 0 {
65596559
t.Errorf(`expected %d got %d`, 0, c)
65606560
}
6561-
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.in`); c != 0 {
6561+
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.in.rangefeed`); c != 0 {
65626562
t.Errorf(`expected 0 got %d`, c)
65636563
}
6564-
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.out`); c != 0 {
6564+
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.out.rangefeed`); c != 0 {
65656565
t.Errorf(`expected 0 got %d`, c)
65666566
}
65676567
if c := s.Server.MustGetSQLCounter(`changefeed.schemafeed.table_metadata_nanos`); c != 0 {
@@ -6594,10 +6594,10 @@ func TestChangefeedMonitoring(t *testing.T) {
65946594
if c := s.Server.MustGetSQLCounter(`changefeed.max_behind_nanos`); c <= 0 {
65956595
return errors.Errorf(`expected > 0 got %d`, c)
65966596
}
6597-
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.in`); c <= 0 {
6597+
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.in.rangefeed`); c <= 0 {
65986598
return errors.Errorf(`expected > 0 got %d`, c)
65996599
}
6600-
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.out`); c <= 0 {
6600+
if c := s.Server.MustGetSQLCounter(`changefeed.buffer_entries.out.rangefeed`); c <= 0 {
66016601
return errors.Errorf(`expected > 0 got %d`, c)
66026602
}
66036603
switch c := s.Server.MustGetSQLCounter(`changefeed.schemafeed.table_history_scans`); {

‎pkg/ccl/changefeedccl/kvevent/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ go_test(
5959
"//pkg/util/hlc",
6060
"//pkg/util/leaktest",
6161
"//pkg/util/log",
62+
"//pkg/util/metric",
6263
"//pkg/util/mon",
6364
"//pkg/util/quotapool",
6465
"//pkg/util/randutil",

‎pkg/ccl/changefeedccl/kvevent/bench_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func BenchmarkMemBuffer(b *testing.B) {
4949
b.ResetTimer()
5050
b.ReportAllocs()
5151

52-
metrics := kvevent.MakeMetrics(time.Minute).AggregatorBufferMetricsWithCompat
52+
metrics := kvevent.MakeMetrics(time.Minute).AggregatorBufferMetrics
5353
st := cluster.MakeTestingClusterSettings()
5454

5555
buf := kvevent.NewMemBuffer(ba, &st.SV, &metrics)

‎pkg/ccl/changefeedccl/kvevent/blocking_buffer.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
// from a mon.BoundAccount and blocks if no resources are available.
2626
type blockingBuffer struct {
2727
sv *settings.Values
28-
metrics *PerBufferMetricsWithCompat
28+
metrics *PerBufferMetrics
2929
qp allocPool // Pool for memory allocations.
3030
signalCh chan struct{} // Signal when new events are available.
3131

@@ -54,7 +54,7 @@ type blockingBuffer struct {
5454
func NewMemBuffer(
5555
acc mon.BoundAccount,
5656
sv *settings.Values,
57-
metrics *PerBufferMetricsWithCompat,
57+
metrics *PerBufferMetrics,
5858
options ...BlockingBufferOption,
5959
) Buffer {
6060
return newMemBuffer(acc, sv, metrics, nil, options...)
@@ -65,7 +65,7 @@ func NewMemBuffer(
6565
func TestingNewMemBuffer(
6666
acc mon.BoundAccount,
6767
sv *settings.Values,
68-
metrics *PerBufferMetricsWithCompat,
68+
metrics *PerBufferMetrics,
6969
onWaitStart quotapool.OnWaitStartFunc,
7070
) Buffer {
7171
return newMemBuffer(acc, sv, metrics, onWaitStart)
@@ -74,7 +74,7 @@ func TestingNewMemBuffer(
7474
func newMemBuffer(
7575
acc mon.BoundAccount,
7676
sv *settings.Values,
77-
metrics *PerBufferMetricsWithCompat,
77+
metrics *PerBufferMetrics,
7878
onWaitStart quotapool.OnWaitStartFunc,
7979
options ...BlockingBufferOption,
8080
) Buffer {
@@ -488,7 +488,7 @@ func (r *memRequest) ShouldWait() bool {
488488

489489
type allocPool struct {
490490
*quotapool.AbstractPool
491-
metrics *PerBufferMetricsWithCompat
491+
metrics *PerBufferMetrics
492492
sv *settings.Values
493493
}
494494

0 commit comments

Comments
 (0)