Skip to content

Commit 21b75ac

Browse files
craig[bot]RaduBerinde
craig[bot]
and
RaduBerinde
committed
Merge #156189
156189: go.mod: bump Pebble to 4cb1291805fb r=RaduBerinde a=RaduBerinde Changes: * [`4cb12918`](cockroachdb/pebble@4cb12918) metrics: fix DiskSpaceUsage after reopen with value sep * [`7d01920a`](cockroachdb/pebble@7d01920a) db: replace preserveBlobReferences * [`fb71b262`](cockroachdb/pebble@fb71b262) db, valsep: replace writeNewBlobFiles with generalizedValueSeparation * [`77a04d51`](cockroachdb/pebble@77a04d51) valsep: introduce new generalized blob writer * [`ed014e0f`](cockroachdb/pebble@ed014e0f) db: fix Metrics.String ingestion count * [`4d123ce7`](cockroachdb/pebble@4d123ce7) metrics: clean up setting of obsolete metrics * [`8bc8cd0e`](cockroachdb/pebble@8bc8cd0e) github: run asan on ubuntu 22.04 * [`89ceab52`](cockroachdb/pebble@89ceab52) metrics: use CountsAndSizes * [`9f32f9ae`](cockroachdb/pebble@9f32f9ae) metrics: use CountAndSize in LevelMetrics * [`b39fdcff`](cockroachdb/pebble@b39fdcff) tool/logs: support virtual sst rewrite compaction summary * [`f04f9389`](cockroachdb/pebble@f04f9389) tool/logs: use ascii package to format summaries * [`e4882db2`](cockroachdb/pebble@e4882db2) tool/logs: support blob file rewrites compaction summary * [`67e0ff09`](cockroachdb/pebble@67e0ff09) options: add Unsafe.AllowMissingWALDir option * [`b4fa5302`](cockroachdb/pebble@b4fa5302) manifest: fix underflow in LevelMetadata * [`aa0103e0`](cockroachdb/pebble@aa0103e0) db: re-export deletepacer.Options Release note: none. Epic: none. Co-authored-by: Radu Berinde <radu@cockroachlabs.com>
2 parents 792292c + ef1c855 commit 21b75ac

File tree

14 files changed

+102
-101
lines changed

14 files changed

+102
-101
lines changed

‎DEPS.bzl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1834,10 +1834,10 @@ def go_deps():
18341834
patches = [
18351835
"@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch",
18361836
],
1837-
sha256 = "6950b51099dbd8abb646135741e4b68b34feaef1f6b653eec1458e35c46ff15e",
1838-
strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20251022203800-ffc306f908df",
1837+
sha256 = "a8e7a07f6fec624733ef5497d2fbe77635ce19f7ed21cc12c4af516fa507e34a",
1838+
strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20251025151421-4cb1291805fb",
18391839
urls = [
1840-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251022203800-ffc306f908df.zip",
1840+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251025151421-4cb1291805fb.zip",
18411841
],
18421842
)
18431843
go_repository(

‎build/bazelutil/distdir_files.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ DISTDIR_FILES = {
352352
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e",
353353
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e",
354354
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea",
355-
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251022203800-ffc306f908df.zip": "6950b51099dbd8abb646135741e4b68b34feaef1f6b653eec1458e35c46ff15e",
355+
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20251025151421-4cb1291805fb.zip": "a8e7a07f6fec624733ef5497d2fbe77635ce19f7ed21cc12c4af516fa507e34a",
356356
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.6.zip": "018eccb5fb9ca52d43ec9eaf213539d01c1f2b94e0e822406ebfb2e9321ef6cf",
357357
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b",
358358
"https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1",

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ require (
137137
github.com/cockroachdb/errors v1.12.1-0.20251010171200-64801262cd6f
138138
github.com/cockroachdb/gostdlib v1.19.0
139139
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506
140-
github.com/cockroachdb/pebble v0.0.0-20251022203800-ffc306f908df
140+
github.com/cockroachdb/pebble v0.0.0-20251025151421-4cb1291805fb
141141
github.com/cockroachdb/redact v1.1.6
142142
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
143143
github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -572,8 +572,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM
572572
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo=
573573
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
574574
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
575-
github.com/cockroachdb/pebble v0.0.0-20251022203800-ffc306f908df h1:PjT+Knqz8an/PiglpXnMxK6APo8d2ktwfXBR16B4dDU=
576-
github.com/cockroachdb/pebble v0.0.0-20251022203800-ffc306f908df/go.mod h1:GflN9NRwberlF+gFRJd6r6jrUYdWR/QD+gtgtH0LRK8=
575+
github.com/cockroachdb/pebble v0.0.0-20251025151421-4cb1291805fb h1:ywJSW1Eg4kpPfAOZ2z+ZCMu639UD3omDqRaeyNPpOVg=
576+
github.com/cockroachdb/pebble v0.0.0-20251025151421-4cb1291805fb/go.mod h1:GflN9NRwberlF+gFRJd6r6jrUYdWR/QD+gtgtH0LRK8=
577577
github.com/cockroachdb/redact v1.1.6 h1:zXJBwDZ84xJNlHl1rMyCojqyIxv+7YUpQiJLQ7n4314=
578578
github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
579579
github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd h1:KFOt5I9nEKZgCnOSmy8r4Oykh8BYQO8bFOTgHDS8YZA=

‎pkg/kv/kvserver/metrics.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4276,7 +4276,7 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
42764276
sm.RdbBloomFilterPrefixChecked.Update(m.Filter.Hits + m.Filter.Misses)
42774277
sm.RdbMemtableTotalSize.Update(int64(m.MemTable.Size))
42784278
sm.RdbFlushes.Update(m.Flush.Count)
4279-
sm.RdbFlushedBytes.Update(int64(m.Levels[0].TableBytesFlushed + m.Levels[0].BlobBytesFlushed))
4279+
sm.RdbFlushedBytes.Update(int64(m.Levels[0].TablesFlushed.Bytes + m.Levels[0].BlobBytesFlushed))
42804280
sm.RdbCompactions.Update(m.Compact.Count)
42814281
sm.RdbIngestedBytes.Update(int64(m.IngestedBytes()))
42824282
compactedRead, compactedWritten := m.CompactedBytes()
@@ -4323,16 +4323,16 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43234323
sm.SecondaryCacheEvictions.Update(m.SecondaryCacheMetrics.Evictions)
43244324
sm.SecondaryCacheWriteBackFails.Update(m.SecondaryCacheMetrics.WriteBackFailures)
43254325
sm.RdbL0Sublevels.Update(int64(m.Levels[0].Sublevels))
4326-
sm.RdbL0NumFiles.Update(m.Levels[0].TablesCount)
4327-
sm.RdbL0BytesFlushed.Update(int64(m.Levels[0].TableBytesFlushed + m.Levels[0].BlobBytesFlushed))
4326+
sm.RdbL0NumFiles.Update(int64(m.Levels[0].Tables.Count))
4327+
sm.RdbL0BytesFlushed.Update(int64(m.Levels[0].TablesFlushed.Bytes + m.Levels[0].BlobBytesFlushed))
43284328
sm.FlushableIngestCount.Update(int64(m.Flush.AsIngestCount))
43294329
sm.FlushableIngestTableCount.Update(int64(m.Flush.AsIngestTableCount))
43304330
sm.FlushableIngestTableSize.Update(int64(m.Flush.AsIngestBytes))
43314331
sm.IngestCount.Update(int64(m.Ingest.Count))
43324332
sm.ValueSeparationBytesReferenced.Update(int64(m.BlobFiles.ReferencedValueSize))
43334333
sm.ValueSeparationBytesUnreferenced.Update(int64(m.BlobFiles.ValueSize - m.BlobFiles.ReferencedValueSize))
4334-
sm.ValueSeparationBlobFileCount.Update(int64(m.BlobFiles.LiveCount))
4335-
sm.ValueSeparationBlobFileSize.Update(int64(m.BlobFiles.LiveSize))
4334+
sm.ValueSeparationBlobFileCount.Update(int64(m.BlobFiles.Live.All.Count))
4335+
sm.ValueSeparationBlobFileSize.Update(int64(m.BlobFiles.Live.All.Bytes))
43364336
sm.ValueSeparationValueRetrievalCount.Update(int64(m.Iterator.ValueRetrievalCount))
43374337
// NB: `UpdateIfHigher` is used here since there is a race in pebble where
43384338
// sometimes the WAL is rotated but metrics are retrieved prior to the update
@@ -4350,10 +4350,10 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43504350
sm.BatchCommitL0StallDuration.Update(int64(m.BatchCommitStats.L0ReadAmpWriteStallDuration))
43514351
sm.BatchCommitWALRotWaitDuration.Update(int64(m.BatchCommitStats.WALRotationDuration))
43524352
sm.BatchCommitCommitWaitDuration.Update(int64(m.BatchCommitStats.CommitWaitDuration))
4353-
sm.SSTableZombieBytes.Update(int64(m.Table.ZombieSize))
4354-
count, size := m.RemoteTablesTotal()
4355-
sm.SSTableRemoteBytes.Update(int64(size))
4356-
sm.SSTableRemoteCount.Update(int64(count))
4353+
sm.SSTableZombieBytes.Update(int64(m.Table.Zombie.All.Bytes))
4354+
remoteTables := m.RemoteTablesTotal()
4355+
sm.SSTableRemoteBytes.Update(int64(remoteTables.Bytes))
4356+
sm.SSTableRemoteCount.Update(int64(remoteTables.Count))
43574357

43584358
c := m.Table.Compression
43594359
c.MergeWith(&m.BlobFiles.Compression)
@@ -4381,8 +4381,8 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43814381
sm.categoryDiskWriteMetrics.update(m.DiskWriteStats)
43824382

43834383
for level, stats := range m.Levels {
4384-
sm.RdbBytesIngested[level].Update(int64(stats.TableBytesIngested))
4385-
sm.RdbLevelSize[level].Update(stats.TablesSize)
4384+
sm.RdbBytesIngested[level].Update(int64(stats.TablesIngested.Bytes))
4385+
sm.RdbLevelSize[level].Update(int64(stats.Tables.Bytes))
43864386
sm.RdbLevelScore[level].Update(stats.Score)
43874387
}
43884388
tot := m.Total()

‎pkg/kv/kvserver/raftstorebench/bench.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func Run(t T, cfg Config) Result {
7979
case <-notifyCh:
8080
n++
8181
l0Metrics := o.raftEng.GetMetrics().Levels[0]
82-
newBytesFlushed := l0Metrics.TableBytesFlushed + l0Metrics.BlobBytesFlushed
82+
newBytesFlushed := l0Metrics.TablesFlushed.Bytes + l0Metrics.BlobBytesFlushed
8383
logf(t, "raft engine flush #%d completed; flushed %s", n,
8484
humanizeutil.IBytes(int64(newBytesFlushed-bytesFlushed)))
8585
bytesFlushed = newBytesFlushed

‎pkg/kv/kvserver/raftstorebench/stats.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,8 +176,8 @@ func writeAmp(m storage.Metrics, payloadBytes int64) WriteAmpStats {
176176
var was WriteAmpStats
177177
was.WALBytesWritten += int64(m.WAL.BytesWritten)
178178
for i := 0; i < len(m.Levels); i++ {
179-
flushed := m.Levels[i].TableBytesFlushed + m.Levels[i].BlobBytesFlushed // only populated for L0
180-
compacted := m.Levels[i].TableBytesCompacted + m.Levels[i].BlobBytesCompacted
179+
flushed := m.Levels[i].TablesFlushed.Bytes + m.Levels[i].BlobBytesFlushed // only populated for L0
180+
compacted := m.Levels[i].TablesCompacted.Bytes + m.Levels[i].BlobBytesCompacted
181181
was.FlushBytesWritten += int64(flushed)
182182
was.CompactionBytesWritten += int64(flushed + compacted)
183183
}

‎pkg/storage/engine.go‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,19 +1344,19 @@ type MetricsForInterval struct {
13441344
// NumSSTables returns the total number of SSTables in the LSM, aggregated
13451345
// across levels.
13461346
func (m *Metrics) NumSSTables() int64 {
1347-
var num int64
1347+
var num uint64
13481348
for _, lm := range m.Metrics.Levels {
1349-
num += lm.TablesCount
1349+
num += lm.Tables.Count
13501350
}
1351-
return num
1351+
return int64(num)
13521352
}
13531353

13541354
// IngestedBytes returns the sum of all ingested tables, aggregated across all
13551355
// levels of the LSM.
13561356
func (m *Metrics) IngestedBytes() uint64 {
13571357
var ingestedBytes uint64
13581358
for _, lm := range m.Metrics.Levels {
1359-
ingestedBytes += lm.TableBytesIngested
1359+
ingestedBytes += lm.TablesIngested.Bytes
13601360
}
13611361
return ingestedBytes
13621362
}
@@ -1366,7 +1366,7 @@ func (m *Metrics) IngestedBytes() uint64 {
13661366
func (m *Metrics) CompactedBytes() (read, written uint64) {
13671367
for _, lm := range m.Metrics.Levels {
13681368
read += lm.TableBytesRead + lm.BlobBytesRead
1369-
written += lm.TableBytesCompacted + lm.BlobBytesCompacted
1369+
written += lm.TablesCompacted.Bytes + lm.BlobBytesCompacted
13701370
}
13711371
return read, written
13721372
}
@@ -1401,32 +1401,32 @@ func (m *Metrics) AsStoreStatsEvent() eventpb.StoreStats {
14011401
WalPhysicalSize: m.WAL.PhysicalSize,
14021402
WalBytesIn: m.WAL.BytesIn,
14031403
WalBytesWritten: m.WAL.BytesWritten,
1404-
TableObsoleteCount: m.Table.ObsoleteCount,
1405-
TableObsoleteSize: m.Table.ObsoleteSize,
1406-
TableZombieCount: m.Table.ZombieCount,
1407-
TableZombieSize: m.Table.ZombieSize,
1404+
TableObsoleteCount: int64(m.Table.Obsolete.All.Count),
1405+
TableObsoleteSize: m.Table.Obsolete.All.Bytes,
1406+
TableZombieCount: int64(m.Table.Zombie.All.Count),
1407+
TableZombieSize: m.Table.Zombie.All.Bytes,
14081408
RangeKeySetsCount: m.Keys.RangeKeySetsCount,
14091409
}
14101410
e.CacheHits, e.CacheMisses = m.BlockCache.HitsAndMisses.Aggregate()
14111411
for i, l := range m.Levels {
1412-
if l.TablesCount == 0 {
1412+
if l.Tables.Count == 0 {
14131413
continue
14141414
}
14151415
e.Levels = append(e.Levels, eventpb.LevelStats{
14161416
Level: uint32(i),
1417-
NumFiles: l.TablesCount,
1418-
SizeBytes: l.TablesSize,
1417+
NumFiles: int64(l.Tables.Count),
1418+
SizeBytes: int64(l.Tables.Bytes),
14191419
Score: float32(l.Score),
14201420
BytesIn: l.TableBytesIn,
1421-
BytesIngested: l.TableBytesIngested,
1422-
BytesMoved: l.TableBytesMoved,
1421+
BytesIngested: l.TablesIngested.Bytes,
1422+
BytesMoved: l.TablesMoved.Bytes,
14231423
BytesRead: l.TableBytesRead + l.BlobBytesRead,
1424-
BytesCompacted: l.TableBytesCompacted + l.BlobBytesCompacted,
1425-
BytesFlushed: l.TableBytesFlushed + l.BlobBytesFlushed,
1426-
TablesCompacted: l.TablesCompacted,
1427-
TablesFlushed: l.TablesFlushed,
1428-
TablesIngested: l.TablesIngested,
1429-
TablesMoved: l.TablesMoved,
1424+
BytesCompacted: l.TablesCompacted.Bytes + l.BlobBytesCompacted,
1425+
BytesFlushed: l.TablesFlushed.Bytes + l.BlobBytesFlushed,
1426+
TablesCompacted: l.TablesCompacted.Count,
1427+
TablesFlushed: l.TablesFlushed.Count,
1428+
TablesIngested: l.TablesIngested.Count,
1429+
TablesMoved: l.TablesMoved.Count,
14301430
NumSublevels: l.Sublevels,
14311431
})
14321432
}
@@ -1756,7 +1756,7 @@ func preIngestDelay(ctx context.Context, eng Engine, settings *cluster.Settings)
17561756
return
17571757
}
17581758
log.VEventf(ctx, 2, "delaying SST ingestion %s. %d L0 files, %d L0 Sublevels",
1759-
targetDelay, metrics.Levels[0].TablesCount, metrics.Levels[0].Sublevels)
1759+
targetDelay, metrics.Levels[0].Tables.Count, metrics.Levels[0].Sublevels)
17601760

17611761
select {
17621762
case <-time.After(targetDelay):
@@ -1769,7 +1769,7 @@ func calculatePreIngestDelay(settings *cluster.Settings, metrics *pebble.Metrics
17691769
l0ReadAmpLimit := ingestDelayL0Threshold.Get(&settings.SV)
17701770

17711771
const ramp = 10
1772-
l0ReadAmp := metrics.Levels[0].TablesCount
1772+
l0ReadAmp := int64(metrics.Levels[0].Tables.Count)
17731773
if metrics.Levels[0].Sublevels >= 0 {
17741774
l0ReadAmp = int64(metrics.Levels[0].Sublevels)
17751775
}

‎pkg/storage/engine_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1195,7 +1195,7 @@ func TestIngestDelayLimit(t *testing.T) {
11951195

11961196
for _, tc := range []struct {
11971197
exp time.Duration
1198-
fileCount int64
1198+
fileCount uint64
11991199
sublevelCount int32
12001200
}{
12011201
{0, 0, 0},
@@ -1208,7 +1208,7 @@ func TestIngestDelayLimit(t *testing.T) {
12081208
{max, 55, -1},
12091209
} {
12101210
var m pebble.Metrics
1211-
m.Levels[0].TablesCount = tc.fileCount
1211+
m.Levels[0].Tables.Count = tc.fileCount
12121212
m.Levels[0].Sublevels = tc.sublevelCount
12131213
require.Equal(t, tc.exp, calculatePreIngestDelay(s, &m))
12141214
}

‎pkg/storage/pebble.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2049,11 +2049,11 @@ func (p *Pebble) GetEnvStats() (*fs.EnvStats, error) {
20492049

20502050
m := p.db.Metrics()
20512051
stats.TotalFiles = 3 /* CURRENT, MANIFEST, OPTIONS */
2052-
stats.TotalFiles += uint64(m.WAL.Files + m.Table.ZombieCount + m.WAL.ObsoleteFiles + m.Table.ObsoleteCount)
2053-
stats.TotalBytes = m.WAL.Size + m.Table.ZombieSize + m.Table.ObsoleteSize
2052+
stats.TotalFiles += uint64(m.WAL.Files) + m.Table.Zombie.All.Count + uint64(m.WAL.ObsoleteFiles) + m.Table.Obsolete.All.Count
2053+
stats.TotalBytes = m.WAL.Size + m.Table.Zombie.All.Bytes + m.Table.Obsolete.All.Bytes
20542054
for _, l := range m.Levels {
2055-
stats.TotalFiles += uint64(l.TablesCount)
2056-
stats.TotalBytes += uint64(l.TablesSize)
2055+
stats.TotalFiles += l.Tables.Count
2056+
stats.TotalBytes += l.Tables.Bytes
20572057
}
20582058

20592059
sstSizes := make(map[pebble.TableNum]uint64)

0 commit comments

Comments
 (0)