Merge pull request #6097 from alexqyle/compactor-execution-id

Add unique execution ID for each compaction cycle in log for easy debugging

Author: danielblando

CHANGELOG.md

@@ -25,6 +25,7 @@
 * [ENHANCEMENT] Ingester: Add link to renew 10% of the ingesters tokens in the admin page. #6063
 * [ENHANCEMENT] Ruler: Add support for filtering by `state` and `health` field on Rules API. #6040
 * [ENHANCEMENT] Ruler: Add support for filtering by `match` field on Rules API. #6083
+* [ENHANCEMENT] Compactor: Add unique execution ID for each compaction cycle in log for easy debugging. #6097
 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920
 * [BUGFIX] Ingester: Fix `user` and `type` labels for the `cortex_ingester_tsdb_head_samples_appended_total` TSDB metric. #5952
 * [BUGFIX] Querier: Enforce max query length check for `/api/v1/series` API even though `ignoreMaxQueryLength` is set to true. #6018

pkg/compactor/compactor.go

@@ -2,6 +2,7 @@ package compactor
 
 import (
 	"context"
+	crypto_rand "crypto/rand"
 	"flag"
 	"fmt"
 	"hash/fnv"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/oklog/ulid"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -810,6 +812,7 @@ func (c *Compactor) compactUser(ctx context.Context, userID string) error {
 	defer c.syncerMetrics.gatherThanosSyncerMetrics(reg)
 
 	ulogger := util_log.WithUserID(userID, c.logger)
+	ulogger = util_log.WithExecutionID(ulid.MustNew(ulid.Now(), crypto_rand.Reader).String(), ulogger)
 
 	// Filters out duplicate blocks that can be formed from two or more overlapping
 	// blocks that fully submatches the source blocks of the older blocks.

pkg/compactor/compactor_test.go

@@ -1629,13 +1629,17 @@ func removeIgnoredLogs(input []string) []string {
 
 	out := make([]string, 0, len(input))
 	durationRe := regexp.MustCompile(`\s?duration(_ms)?=\S+`)
+	executionIDRe := regexp.MustCompile(`\s?execution_id=\S+`)
 
 	for i := 0; i < len(input); i++ {
 		log := input[i]
 
 		// Remove any duration from logs.
 		log = durationRe.ReplaceAllString(log, "")
 
+		// Remove any execution_id from logs.
+		log = executionIDRe.ReplaceAllString(log, "")
+
 		if strings.Contains(log, "block.MetaFetcher") || strings.Contains(log, "block.BaseFetcher") {
 			continue
 		}

pkg/util/log/wrappers.go

@@ -25,6 +25,12 @@ func WithTraceID(traceID string, l log.Logger) log.Logger {
 	return log.With(l, "traceID", traceID)
 }
 
+// WithExecutionID returns a Logger that has information about the execution id in
+// its details.
+func WithExecutionID(executionID string, l log.Logger) log.Logger {
+	return log.With(l, "execution_id", executionID)
+}
+
 // WithContext returns a Logger that has information about the current user in
 // its details.
 //