chore(deps): Migrate routing processor to routing connector for metrics collection

Usage of routing connector is feature flagged and will be kept off/false by default.
After sufficient notices and release notes, the feature flag to use routing connector in
metrics collection will be turned on.

Routing processor is currently removed from the otelcol-contrib version upstream, but
to handle the migration better, sumo is still keeping routing processor in its version
of otel col.

Signed-off-by: Dhruv Shah <dhruv.shah@sumologic.com>

Author: dhruv-shah-sumo

.changelog/4025.added.txt

@@ -0,0 +1 @@
+chore(deps): Migrate routing processor to routing connector for metrics collection

ci/check_configuration_keys.py

@@ -47,6 +47,7 @@
     'sumologic.setup.debug',
     'metrics-server.image.pullSecrets',
     'sumologic.events.sourceCategory',
+    'sumologic.metrics.useRoutingConnectors',
 }
 
 def main(values_path: str, readme_path: str, full_diff=False) -> None:

deploy/helm/sumologic/README.md

@@ -151,6 +151,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.metrics.dropHistogramBuckets`                                              | Drop buckets from histogram and summary metrics, leaving only the sum and count components.                                                                                                                                                                                                                | `true`                                                                                                                                                                               |
 | `sumologic.metrics.allowHistogramRegex`                                               | Allowlist for Histogram metrics, including the buckets                                                                                                                                                                                                                                                     | `"^$"`                                                                                                                                                                               |
 | `sumologic.metrics.sourceType`                                                        | The type of the Sumo Logic source being used for metrics ingestion. Can be `http` or `otlp`.                                                                                                                                                                                                               | `otlp`                                                                                                                                                                               |
+| `sumologic.metrics.useRoutingConnectors`                                              | Enable to use routing connectors.                                                                                                                                                                                                                                                                          | `false`                                                                                                                                                                              |
 | `sumologic.traces.enabled`                                                            | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_                                                                                                                                  | `true`                                                                                                                                                                               |
 | `sumologic.traces.spans_per_request`                                                  | Maximum number of spans sent in single batch                                                                                                                                                                                                                                                               | `100`                                                                                                                                                                                |
 | `sumologic.traces.sourceType`                                                         | The type of the Sumo Logic source being used for traces ingestion. Can be `http` for [HTTP Source](https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/traces/) or `otlp` for [OTLP/HTTP Source](https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/otlp/).       | `otlp`                                                                                                                                                                               |

deploy/helm/sumologic/conf/metrics/otelcol/config.yaml

@@ -1,3 +1,8 @@
+{{- if and (.Values.sumologic.metrics.useRoutingConnectors) (eq .Values.sumologic.metrics.sourceType "http") }}
+connectors:
+{{ tpl (.Files.Get "conf/metrics/otelcol/connectors.yaml") . | indent 2 }}
+{{- end}}
+
 exporters:
 {{ tpl (.Files.Get "conf/metrics/otelcol/exporters.yaml") . | indent 2 }}
 
@@ -53,6 +58,67 @@ service:
   pipelines:
     metrics:
 {{ tpl (.Files.Get "conf/metrics/otelcol/pipeline.yaml") . | indent 6 }}
+{{- if and (.Values.sumologic.metrics.useRoutingConnectors) (eq .Values.sumologic.metrics.sourceType "http") }}
+    metrics/sumologic/default:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    metrics/debug:
+      receivers:
+        - routing/default
+      exporters:
+        - debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    metrics/sumologic/sumologic-mock-default:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/sumologic-mock-default
+    metrics/sumologic/sumologic-mock-http:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/sumologic-mock-http
+{{- end }}
+    metrics/apiserver:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/apiserver
+    metrics/control_plane:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/control_plane
+    metrics/controller:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/controller
+    metrics/kubelet:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/kubelet
+    metrics/node:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/node
+    metrics/scheduler:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/scheduler
+    metrics/state:
+      receivers:
+        - routing/default
+      exporters:
+        - sumologic/state
+{{- end }}
   telemetry:
     logs:
       level: {{ .Values.metadata.metrics.logLevel }}

deploy/helm/sumologic/conf/metrics/otelcol/connectors.yaml

@@ -0,0 +1,82 @@
+routing/default:
+  default_pipelines:
+    - metrics/sumologic/default
+{{- if eq .Values.debug.metrics.metadata.print true }}
+    - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+    - metrics/sumologic/sumologic-mock-default
+{{- end }}
+  table:
+    - statement: route() where resource.attributes["job"] == "apiserver"
+      pipelines:
+        - metrics/apiserver
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "coredns"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-etcd"
+      pipelines:
+        - metrics/control_plane
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-controller-manager"
+      pipelines:
+        - metrics/controller
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kubelet"
+      pipelines:
+        - metrics/kubelet
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "node-exporter"
+      pipelines:
+        - metrics/node
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-scheduler"
+      pipelines:
+        - metrics/scheduler
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}
+    - statement: route() where resource.attributes["job"] == "kube-state-metrics"
+      pipelines:
+        - metrics/state
+{{- if eq .Values.debug.metrics.metadata.print true }}
+        - metrics/debug
+{{- end }}
+{{- if eq (include "sumologic-mock.forward-metrics-metadata" .) "true" }}
+        - metrics/sumologic/sumologic-mock-http
+{{- end }}

deploy/helm/sumologic/conf/metrics/otelcol/pipeline.yaml

@@ -1,4 +1,7 @@
 exporters:
+{{- if and (.Values.sumologic.metrics.useRoutingConnectors) (eq .Values.sumologic.metrics.sourceType "http") }}
+  - routing/default
+{{- else }}
   - sumologic/default
 {{- if eq .Values.debug.metrics.metadata.print true }}
   - debug
@@ -18,6 +21,8 @@ exporters:
   - sumologic/sumologic-mock-http
 {{- end }}
 {{- end }}
+{{- end }}
+
 processors:
   - memory_limiter
   - metricstransform
@@ -45,7 +50,7 @@ processors:
   - filter/app_metrics
 {{- end }}
   - batch
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+{{- if and (not (.Values.sumologic.metrics.useRoutingConnectors | default false)) (eq .Values.sumologic.metrics.sourceType "http") }}
   - routing
 {{- end }}
 receivers:

deploy/helm/sumologic/conf/metrics/otelcol/processors.yaml

@@ -165,7 +165,7 @@ resource/remove_k8s_pod_pod_name:
     - action: delete
       key: k8s.pod.pod_name
 
-{{- if eq .Values.sumologic.metrics.sourceType "http" }}
+{{- if and (not (.Values.sumologic.metrics.useRoutingConnectors | default false)) (eq .Values.sumologic.metrics.sourceType "http") }}
 routing:
   default_exporters:
     - sumologic/default

deploy/helm/sumologic/values.yaml

@@ -462,11 +462,14 @@ sumologic:
         fallbackExporters: []
         table: []
         ## -
-        ##  ## exporter is name of the exporter
-        ##  exporter: sumologic/otlp
-        ##  ## statement is an OTTL condition which defines what data should be send to the exporter
-        ##  ## see routing processor documentation for more details:
-        ##  ## https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/aee4b75100530bce7edbf736fbcf76ac4f6ced6d/processor/routingprocessor/README.md#tech-preview-opentelemetry-transformation-language-statements-as-routing-conditions
+        ##  ## exporters is an array of the exporter
+        ##  exporters:
+        ##    - sumologic/otlp
+        ##  ## statement is an OTTL condition which defines what data should be sent to the exporters
+        ##  ## see routing connector documentation for more details:
+        ##  ## https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/aee4b75100530bce7edbf736fbcf76ac4f6ced6d/connector/routingconnector
+        ## routing connector expects you to define a pipeline, but here the user can just mention the exporter.
+        ## The helm deployment would convert exporters into correct pipelines and configure the connectors.
         ##  statement: ""
 
     ## Fields to be created at Sumo Logic to ensure logs are tagged with
@@ -764,6 +767,8 @@ sumologic:
     ## The type of source we send to in Sumo. The possible values are http and otlp.
     ## Consult the documentation for more information.
     sourceType: otlp
+    ## In order to use routing connectors, please enable this flag.
+    # useRoutingConnectors: true
 
   ### Traces configuration
   ## Set the enabled flag to false to disable traces from instrumentation ingestion.

tests/helm/metrics_test.go

@@ -155,6 +155,7 @@ func TestMetadataSourceTypeOTLP(t *testing.T) {
 			Rest map[string]interface{} `yaml:",inline"`
 		}
 		Processors map[string]interface{}
+		Connectors map[string]interface{}
 		Service    struct {
 			Pipelines struct {
 				Metrics struct {
@@ -179,6 +180,7 @@ sumologic:
 	assert.Equal(t, otelConfig.Exporters.Default.Endpoint, "${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}")
 	assert.Len(t, otelConfig.Exporters.Rest, 0)
 	assert.NotContains(t, otelConfig.Processors, "routing")
+	assert.NotContains(t, otelConfig.Connectors, "routing")
 	assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
 	assert.Equal(t, otelConfig.Service.Pipelines.Metrics.Exporters, []string{"sumologic/default"})
 }
@@ -187,18 +189,32 @@ func TestMetadataSourceTypeHTTP(t *testing.T) {
 	t.Parallel()
 	templatePath := "templates/metrics/otelcol/configmap.yaml"
 
+	type MetricsPipeline struct {
+		Receivers []string `yaml:"receivers,omitempty"`
+		Exporters []string `yaml:"exporters,omitempty"`
+	}
+
 	type OtelConfig struct {
 		Exporters map[string]struct {
 			MetricFormat string `yaml:"metric_format"`
 			Endpoint     string
 		} `yaml:"exporters"`
 		Processors map[string]interface{}
+		Connectors map[string]interface{}
 		Service    struct {
 			Pipelines struct {
 				Metrics struct {
 					Processors []string `yaml:"processors"`
 					Exporters  []string `yaml:"exporters"`
 				}
+				MetricsApiserver        MetricsPipeline `yaml:"metrics/apiserver"`
+				MetricsControlPlane     MetricsPipeline `yaml:"metrics/control_plane"`
+				MetricsController       MetricsPipeline `yaml:"metrics/controller"`
+				MetricsKubelet          MetricsPipeline `yaml:"metrics/kubelet"`
+				MetricsNode             MetricsPipeline `yaml:"metrics/node"`
+				MetricsScheduler        MetricsPipeline `yaml:"metrics/scheduler"`
+				MetricsState            MetricsPipeline `yaml:"metrics/state"`
+				MetricsSumologicDefault MetricsPipeline `yaml:"metrics/sumologic/default"`
 			}
 		}
 	}
@@ -208,6 +224,7 @@ func TestMetadataSourceTypeHTTP(t *testing.T) {
 sumologic:
   metrics:
     sourceType: http
+    useRoutingConnectors: true
 `
 	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
 	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
@@ -217,6 +234,65 @@ sumologic:
 	defaultExporter := otelConfig.Exporters["sumologic/default"]
 	assert.Equal(t, "prometheus", defaultExporter.MetricFormat)
 	assert.Equal(t, "${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}", defaultExporter.Endpoint)
+	assert.Contains(t, otelConfig.Connectors, "routing/default")
+	assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
+	assert.Equal(t, otelConfig.Service.Pipelines.Metrics.Exporters, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsApiserver.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsControlPlane.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsController.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsKubelet.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsNode.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsScheduler.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsState.Receivers, []string{"routing/default"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsSumologicDefault.Receivers, []string{"routing/default"})
+
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsApiserver.Exporters, []string{"sumologic/apiserver"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsControlPlane.Exporters, []string{"sumologic/control_plane"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsController.Exporters, []string{"sumologic/controller"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsKubelet.Exporters, []string{"sumologic/kubelet"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsNode.Exporters, []string{"sumologic/node"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsScheduler.Exporters, []string{"sumologic/scheduler"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsState.Exporters, []string{"sumologic/state"})
+	assert.Equal(t, otelConfig.Service.Pipelines.MetricsSumologicDefault.Exporters, []string{"sumologic/default"})
+
+	// useRoutingConnectors: false which means routing processor should be used.
+	valuesYaml = `
+sumologic:
+  metrics:
+    sourceType: http
+    useRoutingConnectors: false
+`
+	otelConfigYaml = GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err = yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+
+	assert.Contains(t, otelConfig.Processors, "routing")
+	assert.Contains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
+	assert.Equal(
+		t,
+		[]string{
+			"sumologic/default",
+			"sumologic/apiserver",
+			"sumologic/control_plane",
+			"sumologic/controller",
+			"sumologic/kubelet",
+			"sumologic/node",
+			"sumologic/scheduler",
+			"sumologic/state",
+		},
+		otelConfig.Service.Pipelines.Metrics.Exporters,
+	)
+
+	// useRoutingConnectors flag not present, which means routing processor should be used.
+	valuesYaml = `
+sumologic:
+  metrics:
+    sourceType: http
+`
+	otelConfigYaml = GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err = yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+
 	assert.Contains(t, otelConfig.Processors, "routing")
 	assert.Contains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
 	assert.Equal(

tests/helm/testdata/goldenfile/metadata_metrics_otc/custom_routing_connector.input.yaml

@@ -0,0 +1,13 @@
+sumologic:
+  ipv6mode: true
+  metrics:
+    sourceType: http
+    useRoutingConnectors: true
+metadata:
+  metrics:
+    autoscaling:
+      enabled: false
+debug:
+  metrics:
+    metadata:
+      print: true