Add -distributor.enable-type-and-unit-labels per tenant flag

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

Author: SungJin1212

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master / unreleased
 
+* [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` which enables to add `__unit__` and `__type__` labels for remote write v2 and OTLP requests. #7077
 * [ENHANCEMENT] Ingester: Add `enable_matcher_optimization` config to apply low selectivity matchers lazily. #7063
 * [BUGFIX] Compactor: Avoid race condition which allow a grouper to not compact all partitions. #7082
 
@@ -30,7 +31,6 @@
 * [FEATURE] Querier: Support for configuring query optimizers and enabling XFunctions in the Thanos engine. #6873
 * [FEATURE] Query Frontend: Add support /api/v1/format_query API for formatting queries. #6893
 * [FEATURE] Query Frontend: Add support for /api/v1/parse_query API (experimental) to parse a PromQL expression and return it as a JSON-formatted AST (abstract syntax tree). #6978
-* [FEATURE] Distributor: Add a per-tenant flag `-distributor.rw2-enable-type-and-unit-labels` which enables to add `__unit__` and `__type__` labels for remote write v2 requests. #7077
 * [ENHANCEMENT] Upgrade the Prometheus version to 3.6.0 and add a `-name-validation-scheme` flag to support UTF-8. #7040 #7056
 * [ENHANCEMENT] Distributor: Emit an error with a 400 status code when empty labels are found before the relabelling or label dropping process. #7052
 * [ENHANCEMENT] Parquet Storage: Add support for additional sort columns during Parquet file generation #7003

docs/configuration/config-file-reference.md

@@ -3270,11 +3270,6 @@ otlp:
   # EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.
   # CLI flag: -distributor.otlp.allow-delta-temporality
   [allow_delta_temporality: <boolean> | default = false]
-
-  # EXPERIMENTAL: If true, the '__type__' and '__unit__' labels are added for
-  # the OTLP metrics.
-  # CLI flag: -distributor.otlp.enable-type-and-unit-labels
-  [enable_type_and_unit_labels: <boolean> | default = false]
 ```
 
 ### `etcd_config`
@@ -3995,9 +3990,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 [promote_resource_attributes: <list of string> | default = ]
 
 # EXPERIMENTAL: If true, the __type__ and __unit__ labels are added to metrics.
-# This only applies to remote write v2 requests.
-# CLI flag: -distributor.rw2-enable-type-and-unit-labels
-[rw_2_enable_type_and_unit_labels: <boolean> | default = false]
+# This applies to remote write v2 and OTLP requests.
+# CLI flag: -distributor.enable-type-and-unit-labels
+[enable_type_and_unit_labels: <boolean> | default = false]
 
 # The maximum number of active series per user, per ingester. 0 to disable.
 # CLI flag: -ingester.max-series-per-user

integration/remote_write_v2_test.go

@@ -234,9 +234,9 @@ func TestIngest_EnableTypeAndUnitLabels(t *testing.T) {
 			"-ring.store":      "consul",
 			"-consul.hostname": consul.NetworkHTTPEndpoint(),
 			// Distributor.
-			"-distributor.replication-factor":              "1",
-			"-distributor.remote-writev2-enabled":          "true",
-			"-distributor.rw2-enable-type-and-unit-labels": "true",
+			"-distributor.replication-factor":          "1",
+			"-distributor.remote-writev2-enabled":      "true",
+			"-distributor.enable-type-and-unit-labels": "true",
 			// Store-gateway.
 			"-store-gateway.sharding-enabled": "false",
 			// alert manager

pkg/distributor/distributor.go

@@ -197,10 +197,9 @@ type InstanceLimits struct {
 }
 
 type OTLPConfig struct {
-	ConvertAllAttributes    bool `yaml:"convert_all_attributes"`
-	DisableTargetInfo       bool `yaml:"disable_target_info"`
-	AllowDeltaTemporality   bool `yaml:"allow_delta_temporality"`
-	EnableTypeAndUnitLabels bool `yaml:"enable_type_and_unit_labels"`
+	ConvertAllAttributes  bool `yaml:"convert_all_attributes"`
+	DisableTargetInfo     bool `yaml:"disable_target_info"`
+	AllowDeltaTemporality bool `yaml:"allow_delta_temporality"`
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
@@ -229,7 +228,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.OTLPConfig.ConvertAllAttributes, "distributor.otlp.convert-all-attributes", false, "If true, all resource attributes are converted to labels.")
 	f.BoolVar(&cfg.OTLPConfig.DisableTargetInfo, "distributor.otlp.disable-target-info", false, "If true, a target_info metric is not ingested. (refer to: https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)")
 	f.BoolVar(&cfg.OTLPConfig.AllowDeltaTemporality, "distributor.otlp.allow-delta-temporality", false, "EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.")
-	f.BoolVar(&cfg.OTLPConfig.EnableTypeAndUnitLabels, "distributor.otlp.enable-type-and-unit-labels", false, "EXPERIMENTAL: If true, the '__type__' and '__unit__' labels are added for the OTLP metrics.")
 }
 
 // Validate config and returns error on failure

pkg/util/push/otlp.go

@@ -181,7 +181,7 @@ func convertToPromTS(ctx context.Context, pmetrics pmetric.Metrics, cfg distribu
 		AddMetricSuffixes:       true,
 		DisableTargetInfo:       cfg.DisableTargetInfo,
 		AllowDeltaTemporality:   cfg.AllowDeltaTemporality,
-		EnableTypeAndUnitLabels: cfg.EnableTypeAndUnitLabels,
+		EnableTypeAndUnitLabels: overrides.EnableTypeAndUnitLabels(userID),
 	}
 
 	var annots annotations.Annotations

pkg/util/push/otlp_test.go

@@ -80,8 +80,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
 			cfg := distributor.OTLPConfig{
-				EnableTypeAndUnitLabels: test.enableTypeAndUnitLabels,
-				AllowDeltaTemporality:   test.allowDeltaTemporality,
+				AllowDeltaTemporality: test.allowDeltaTemporality,
 			}
 			metrics := pmetric.NewMetrics()
 			rm := metrics.ResourceMetrics().AppendEmpty()
@@ -90,6 +89,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 			test.otlpSeries.CopyTo(sm.Metrics().AppendEmpty())
 
 			limits := validation.Limits{}
+			limits.EnableTypeAndUnitLabels = test.enableTypeAndUnitLabels
 			overrides := validation.NewOverrides(limits, nil)
 			promSeries, metadata, err := convertToPromTS(ctx, metrics, cfg, overrides, "user-1", logger)
 			require.NoError(t, err)

pkg/util/push/push.go

@@ -99,7 +99,7 @@ func Handler(remoteWrite2Enabled bool, maxRecvMsgSize int, overrides *validation
 				req.Source = cortexpb.API
 			}
 
-			v1Req, err := convertV2RequestToV1(&req, overrides.RW2EnableTypeAndUnitLabels(userID))
+			v1Req, err := convertV2RequestToV1(&req, overrides.EnableTypeAndUnitLabels(userID))
 			if err != nil {
 				level.Error(logger).Log("err", err.Error())
 				http.Error(w, err.Error(), http.StatusBadRequest)

pkg/util/validation/limits.go

@@ -152,7 +152,7 @@ type Limits struct {
 	MetricRelabelConfigs              []*relabel.Config   `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs."`
 	MaxNativeHistogramBuckets         int                 `yaml:"max_native_histogram_buckets" json:"max_native_histogram_buckets"`
 	PromoteResourceAttributes         []string            `yaml:"promote_resource_attributes" json:"promote_resource_attributes"`
-	RW2EnableTypeAndUnitLabels        bool                `yaml:"rw_2_enable_type_and_unit_labels" json:"rw_2_enable_type_and_unit_labels"`
+	EnableTypeAndUnitLabels           bool                `yaml:"enable_type_and_unit_labels" json:"enable_type_and_unit_labels"`
 
 	// Ingester enforced limits.
 	// Series
@@ -265,7 +265,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.HAMaxClusters, "distributor.ha-tracker.max-clusters", 0, "Maximum number of clusters that HA tracker will keep track of for single user. 0 to disable the limit.")
 	f.Var((*flagext.StringSliceCSV)(&l.PromoteResourceAttributes), "distributor.promote-resource-attributes", "Comma separated list of resource attributes that should be converted to labels.")
 	f.Var(&l.DropLabels, "distributor.drop-label", "This flag can be used to specify label names that to drop during sample ingestion within the distributor and can be repeated in order to drop multiple labels.")
-	f.BoolVar(&l.RW2EnableTypeAndUnitLabels, "distributor.rw2-enable-type-and-unit-labels", false, "EXPERIMENTAL: If true, the __type__ and __unit__ labels are added to metrics. This only applies to remote write v2 requests.")
+	f.BoolVar(&l.EnableTypeAndUnitLabels, "distributor.enable-type-and-unit-labels", false, "EXPERIMENTAL: If true, the __type__ and __unit__ labels are added to metrics. This applies to remote write v2 and OTLP requests.")
 	f.IntVar(&l.MaxLabelNameLength, "validation.max-length-label-name", 1024, "Maximum length accepted for label names")
 	f.IntVar(&l.MaxLabelValueLength, "validation.max-length-label-value", 2048, "Maximum length accepted for label value. This setting also applies to the metric name")
 	f.IntVar(&l.MaxLabelNamesPerSeries, "validation.max-label-names-per-series", 30, "Maximum number of label names per series.")
@@ -1094,8 +1094,8 @@ func (o *Overrides) AlertmanagerMaxSilenceSizeBytes(userID string) int {
 	return o.GetOverridesForUser(userID).AlertmanagerMaxSilencesSizeBytes
 }
 
-func (o *Overrides) RW2EnableTypeAndUnitLabels(userID string) bool {
-	return o.GetOverridesForUser(userID).RW2EnableTypeAndUnitLabels
+func (o *Overrides) EnableTypeAndUnitLabels(userID string) bool {
+	return o.GetOverridesForUser(userID).EnableTypeAndUnitLabels
 }
 
 func (o *Overrides) DisabledRuleGroups(userID string) DisabledRuleGroups {

schemas/cortex-config-schema.json

@@ -3816,12 +3816,6 @@
               "description": "If true, a target_info metric is not ingested. (refer to: https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)",
               "type": "boolean",
               "x-cli-flag": "distributor.otlp.disable-target-info"
-            },
-            "enable_type_and_unit_labels": {
-              "default": false,
-              "description": "EXPERIMENTAL: If true, the '__type__' and '__unit__' labels are added for the OTLP metrics.",
-              "type": "boolean",
-              "x-cli-flag": "distributor.otlp.enable-type-and-unit-labels"
             }
           },
           "type": "object"
@@ -4940,6 +4934,12 @@
           "type": "boolean",
           "x-cli-flag": "blocks-storage.tsdb.enable-native-histograms"
         },
+        "enable_type_and_unit_labels": {
+          "default": false,
+          "description": "EXPERIMENTAL: If true, the __type__ and __unit__ labels are added to metrics. This applies to remote write v2 and OTLP requests.",
+          "type": "boolean",
+          "x-cli-flag": "distributor.enable-type-and-unit-labels"
+        },
         "enforce_metadata_metric_name": {
           "default": true,
           "description": "Enforce every metadata has a metric name.",
@@ -5369,12 +5369,6 @@
           "description": "Enable to allow rules to be evaluated with data from a single zone, if other zones are not available.",
           "type": "boolean"
         },
-        "rw_2_enable_type_and_unit_labels": {
-          "default": false,
-          "description": "EXPERIMENTAL: If true, the __type__ and __unit__ labels are added to metrics. This only applies to remote write v2 requests.",
-          "type": "boolean",
-          "x-cli-flag": "distributor.rw2-enable-type-and-unit-labels"
-        },
         "s3_sse_kms_encryption_context": {
           "description": "S3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set.",
           "type": "string"