Skip to content

Commit 20baea9

Browse files
CharlieTLeCharlieTLe
authored
Merge pull request #6550 from CharlieTLe/prepare-release-1.19-rc.1
Prepare release 1.19.0-rc.1
2 parents 5f4c165 + 87c7627 commit 20baea9

File tree

3 files changed

+133
-12
lines changed

3 files changed

+133
-12
lines changed

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.19.0-rc.0
1+
1.19.0-rc.1

‎pkg/compactor/partition_compaction_grouper.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -499,9 +499,11 @@ func (g *PartitionCompactionGrouper) partitionBlocksGroup(partitionCount int, bl
499499
addToPartitionedGroups := func(blocks []*metadata.Meta, partitionID int) {
500500
if _, ok := partitionedGroups[partitionID]; !ok {
501501
partitionedGroups[partitionID] = blocksGroupWithPartition{
502-
rangeStart: rangeStart,
503-
rangeEnd: rangeEnd,
504-
blocks: []*metadata.Meta{},
502+
blocksGroup: blocksGroup{
503+
rangeStart: rangeStart,
504+
rangeEnd: rangeEnd,
505+
blocks: []*metadata.Meta{},
506+
},
505507
}
506508
}
507509
partitionedGroup := partitionedGroups[partitionID]
@@ -868,9 +870,6 @@ func (t *timeRangeStatus) previousTimeRangeDuration() time.Duration {
868870

869871
type blocksGroupWithPartition struct {
870872
blocksGroup
871-
rangeStart int64 // Included.
872-
rangeEnd int64 // Excluded.
873-
blocks []*metadata.Meta
874873
groupHash uint32
875874
partitionedGroupInfo *PartitionedGroupInfo
876875
partition Partition

‎pkg/compactor/partition_compaction_grouper_test.go‎

Lines changed: 127 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,64 @@ func TestPartitionCompactionGrouper_GenerateCompactionJobs(t *testing.T) {
8484
{blocks: []ulid.ULID{block3, block4}, partitionCount: 1, partitionID: 0, rangeStart: 2 * H, rangeEnd: 4 * H},
8585
},
8686
},
87+
"only level 1 blocks with ingestion replication factor 3": {
88+
ranges: []time.Duration{2 * time.Hour, 12 * time.Hour, 24 * time.Hour},
89+
blocks: map[ulid.ULID]mockBlock{
90+
block1: {
91+
meta: &metadata.Meta{
92+
BlockMeta: tsdb.BlockMeta{ULID: block1, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
93+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
94+
},
95+
timeRange: 2 * time.Hour,
96+
hasNoCompactMark: false,
97+
},
98+
block2: {
99+
meta: &metadata.Meta{
100+
BlockMeta: tsdb.BlockMeta{ULID: block2, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
101+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
102+
},
103+
timeRange: 2 * time.Hour,
104+
hasNoCompactMark: false,
105+
},
106+
block3: {
107+
meta: &metadata.Meta{
108+
BlockMeta: tsdb.BlockMeta{ULID: block3, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
109+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
110+
},
111+
timeRange: 2 * time.Hour,
112+
hasNoCompactMark: false,
113+
},
114+
block4: {
115+
meta: &metadata.Meta{
116+
BlockMeta: tsdb.BlockMeta{ULID: block4, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
117+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
118+
},
119+
timeRange: 2 * time.Hour,
120+
hasNoCompactMark: false,
121+
},
122+
block5: {
123+
meta: &metadata.Meta{
124+
BlockMeta: tsdb.BlockMeta{ULID: block5, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
125+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
126+
},
127+
timeRange: 2 * time.Hour,
128+
hasNoCompactMark: false,
129+
},
130+
block6: {
131+
meta: &metadata.Meta{
132+
BlockMeta: tsdb.BlockMeta{ULID: block6, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 1}, Stats: tsdb.BlockStats{NumSeries: 1}},
133+
Thanos: metadata.Thanos{Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
134+
},
135+
timeRange: 2 * time.Hour,
136+
hasNoCompactMark: false,
137+
},
138+
},
139+
existingPartitionedGroups: []mockExistingPartitionedGroup{},
140+
expected: []expectedCompactionJob{
141+
{blocks: []ulid.ULID{block1, block2, block3, block4, block5, block6}, partitionCount: 1, partitionID: 0, rangeStart: 0 * H, rangeEnd: 2 * H},
142+
},
143+
ingestionReplicationFactor: 3,
144+
},
87145
"only level 1 blocks, there is existing partitioned group file": {
88146
ranges: []time.Duration{2 * time.Hour, 12 * time.Hour, 24 * time.Hour},
89147
blocks: map[ulid.ULID]mockBlock{
@@ -499,6 +557,65 @@ func TestPartitionCompactionGrouper_GenerateCompactionJobs(t *testing.T) {
499557
{blocks: []ulid.ULID{block1, block2, block3}, partitionCount: 1, partitionID: 0, rangeStart: 0 * H, rangeEnd: 12 * H},
500558
},
501559
},
560+
"level 2 blocks with ingestion replication factor 3": {
561+
ranges: []time.Duration{2 * time.Hour, 12 * time.Hour, 24 * time.Hour},
562+
blocks: map[ulid.ULID]mockBlock{
563+
block1: {
564+
meta: &metadata.Meta{
565+
BlockMeta: tsdb.BlockMeta{ULID: block1, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
566+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 0}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
567+
},
568+
timeRange: 2 * time.Hour,
569+
hasNoCompactMark: false,
570+
},
571+
block2: {
572+
meta: &metadata.Meta{
573+
BlockMeta: tsdb.BlockMeta{ULID: block2, MinTime: 0 * H, MaxTime: 2 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
574+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 1}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
575+
},
576+
timeRange: 2 * time.Hour,
577+
hasNoCompactMark: false,
578+
},
579+
block3: {
580+
meta: &metadata.Meta{
581+
BlockMeta: tsdb.BlockMeta{ULID: block3, MinTime: 2 * H, MaxTime: 4 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
582+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 0}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
583+
},
584+
timeRange: 2 * time.Hour,
585+
hasNoCompactMark: false,
586+
},
587+
block4: {
588+
meta: &metadata.Meta{
589+
BlockMeta: tsdb.BlockMeta{ULID: block4, MinTime: 2 * H, MaxTime: 4 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
590+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 1}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
591+
},
592+
timeRange: 2 * time.Hour,
593+
hasNoCompactMark: false,
594+
},
595+
block5: {
596+
meta: &metadata.Meta{
597+
BlockMeta: tsdb.BlockMeta{ULID: block5, MinTime: 4 * H, MaxTime: 6 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
598+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 0}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
599+
},
600+
timeRange: 2 * time.Hour,
601+
hasNoCompactMark: false,
602+
},
603+
block6: {
604+
meta: &metadata.Meta{
605+
BlockMeta: tsdb.BlockMeta{ULID: block6, MinTime: 4 * H, MaxTime: 6 * H, Compaction: tsdb.BlockMetaCompaction{Level: 2}, Stats: tsdb.BlockStats{NumSeries: 1}},
606+
Thanos: metadata.Thanos{Extensions: cortextsdb.CortexMetaExtensions{PartitionInfo: &cortextsdb.PartitionInfo{PartitionCount: 2, PartitionID: 1}}, Files: []metadata.File{{RelPath: thanosblock.IndexFilename, SizeBytes: 0}}},
607+
},
608+
timeRange: 2 * time.Hour,
609+
hasNoCompactMark: false,
610+
},
611+
},
612+
existingPartitionedGroups: []mockExistingPartitionedGroup{},
613+
expected: []expectedCompactionJob{
614+
{blocks: []ulid.ULID{block1, block3, block5}, partitionCount: 2, partitionID: 0, rangeStart: 0 * H, rangeEnd: 12 * H},
615+
{blocks: []ulid.ULID{block2, block4, block6}, partitionCount: 2, partitionID: 1, rangeStart: 0 * H, rangeEnd: 12 * H},
616+
},
617+
ingestionReplicationFactor: 3,
618+
},
502619
"level 2 blocks along with level 3 blocks from some of partitions, level 1 blocks in different time range, there are partitioned group files for all groups": {
503620
ranges: []time.Duration{2 * time.Hour, 12 * time.Hour, 24 * time.Hour},
504621
blocks: map[ulid.ULID]mockBlock{
@@ -1966,6 +2083,10 @@ func TestPartitionCompactionGrouper_GenerateCompactionJobs(t *testing.T) {
19662083

19672084
ctx, cancel := context.WithCancel(context.Background())
19682085
defer cancel()
2086+
ingestionReplicationFactor := 1
2087+
if testCase.ingestionReplicationFactor > 1 {
2088+
ingestionReplicationFactor = testCase.ingestionReplicationFactor
2089+
}
19692090
g := NewPartitionCompactionGrouper(
19702091
ctx,
19712092
nil,
@@ -1988,7 +2109,7 @@ func TestPartitionCompactionGrouper_GenerateCompactionJobs(t *testing.T) {
19882109
false,
19892110
visitMarkerTimeout,
19902111
noCompactFilter,
1991-
1,
2112+
ingestionReplicationFactor,
19922113
)
19932114
actual, err := g.generateCompactionJobs(testCase.getBlocks())
19942115
require.NoError(t, err)
@@ -2011,10 +2132,11 @@ func TestPartitionCompactionGrouper_GenerateCompactionJobs(t *testing.T) {
20112132
}
20122133

20132134
type generateCompactionJobsTestCase struct {
2014-
ranges []time.Duration
2015-
blocks map[ulid.ULID]mockBlock
2016-
existingPartitionedGroups []mockExistingPartitionedGroup
2017-
expected []expectedCompactionJob
2135+
ranges []time.Duration
2136+
blocks map[ulid.ULID]mockBlock
2137+
existingPartitionedGroups []mockExistingPartitionedGroup
2138+
expected []expectedCompactionJob
2139+
ingestionReplicationFactor int
20182140
}
20192141

20202142
func (g *generateCompactionJobsTestCase) setupBucketStore(t *testing.T, bkt *bucket.ClientMock, userID string, visitMarkerTimeout time.Duration) {

0 commit comments

Comments
 (0)