OpenTelemetry improvements (#257)

* Use FastAPI instrumentor

* Calculate RED metrics from spans

* Default to local opentelemetry export

* Switch to manual instrumentation

Author: febus982

.idea/runConfigurations/Otel_Stack.xml

@@ -80,19 +80,19 @@ FROM base_app AS http
 COPY --from=http_builder /venv /venv
 COPY --chown=nonroot:nonroot src/http_app ./http_app
 # Run CMD using array syntax, so it uses `exec` and runs as PID1
-CMD ["opentelemetry-instrument", "python", "-m", "http_app"]
+CMD ["python", "-m", "http_app"]
 
 # Copy the socketio python package and requirements from relevant builder
 FROM base_app AS socketio
 COPY --from=socketio_builder /venv /venv
 COPY --chown=nonroot:nonroot src/socketio_app ./socketio_app
 # Run CMD using array syntax, so it uses `exec` and runs as PID1
-CMD ["opentelemetry-instrument", "python", "-m", "socketio_app"]
+CMD ["python", "-m", "socketio_app"]
 
 # Copy the dramatiq python package and requirements from relevant builder
 FROM base_app AS dramatiq
 COPY --from=dramatiq_builder /venv /venv
 COPY --chown=nonroot:nonroot src/dramatiq_worker ./dramatiq_worker
 # Run CMD using array syntax, so it uses `exec` and runs as PID1
 # TODO: Review processes/threads
-CMD ["opentelemetry-instrument", "dramatiq", "-p", "1", "-t", "1", "dramatiq_worker"]
+CMD ["dramatiq", "-p", "1", "-t", "1", "dramatiq_worker"]

Dockerfile

@@ -1,11 +1,6 @@
 .PHONY: docs docs-build adr
 
 containers:
-	# Use local UID to avoid files permission issues when mounting directories
-	# We could do this at runtime, by specifying the user, but it's easier doing it
-	# at build time, so no special commands will be necessary at runtime
-	docker compose build --build-arg UID=`id -u` dev
-	# To build shared container layers only once we build a single container before the other ones
 	docker compose build --build-arg UID=`id -u`
 
 dev-http:
@@ -14,9 +9,6 @@ dev-http:
 dev-socketio:
 	uv run ./src/socketio_app/dev_server.py
 
-otel:
-	OTEL_SERVICE_NAME=bootstrap-fastapi OTEL_TRACES_EXPORTER=none OTEL_METRICS_EXPORTER=none OTEL_LOGS_EXPORTER=none uv run opentelemetry-instrument uvicorn http_app:create_app --host 0.0.0.0 --port 8000 --factory
-
 run:
 	uv run uvicorn http_app:create_app --host 0.0.0.0 --port 8000 --factory
 

Makefile

@@ -1,3 +1,85 @@
+otelcol.receiver.otlp "default" {
+	grpc { }
+
+	output {
+		metrics = [
+		    otelcol.processor.transform.add_resource_attributes_as_metric_attributes.input,
+        ]
+		traces  = [
+		    // This transforms the traces in metrics, we still have to send traces out
+            otelcol.connector.spanmetrics.asgi_apm.input,
+		    // This also transforms the traces in metrics, we still have to send traces out
+		    otelcol.connector.host_info.default.input,
+		    // This sends the traces out
+            otelcol.processor.batch.default.input,
+		]
+		logs    = [
+		    otelcol.processor.batch.default.input,
+		]
+	}
+}
+
+otelcol.connector.host_info "default" {
+  // https://grafana.com/docs/alloy/latest/reference/components/otelcol.connector.host_info/
+  host_identifiers = ["host.name"]
+
+  output {
+    metrics = [otelcol.processor.batch.default.input]
+  }
+}
+
+otelcol.connector.spanmetrics "asgi_apm" {
+  dimension {
+    name = "http.status_code"
+  }
+
+  dimension {
+    name = "http.method"
+  }
+
+  dimension {
+    name = "http.route"
+  }
+
+  histogram {
+    explicit {
+        buckets = ["2ms", "4ms", "6ms", "8ms", "10ms", "50ms", "100ms", "200ms", "400ms", "800ms", "1s", "1400ms", "2s", "5s", "10s", "15s"]
+    }
+  }
+
+  output {
+    metrics = [otelcol.processor.transform.add_resource_attributes_as_metric_attributes.input]
+  }
+}
+
+otelcol.processor.transform "add_resource_attributes_as_metric_attributes" {
+	error_mode = "ignore"
+
+	metric_statements {
+		context    = "datapoint"
+		statements = [
+			"set(attributes[\"deployment.environment\"], resource.attributes[\"deployment.environment\"])",
+			"set(attributes[\"service.version\"], resource.attributes[\"service.version\"])",
+		]
+	}
+
+	output {
+        metrics = [otelcol.processor.batch.default.input]
+    }
+}
+
+otelcol.processor.batch "default" {
+	output {
+		// metrics = [otelcol.exporter.otlphttp.grafanacloud.input]
+		// logs    = [otelcol.exporter.otlphttp.grafanacloud.input]
+		// traces  = [otelcol.exporter.otlphttp.grafanacloud.input]
+
+		metrics = [otelcol.exporter.debug.console.input]
+		logs    = [otelcol.exporter.debug.console.input]
+		traces  = [otelcol.exporter.otlp.jaeger.input]
+	}
+}
+
 otelcol.exporter.otlp "jaeger" {
 	client {
 		endpoint = "jaeger:4317"
@@ -12,18 +94,15 @@ otelcol.exporter.debug "console" {
 	verbosity = "Detailed"
 }
 
-otelcol.processor.batch "default" {
-	output {
-		traces = [otelcol.exporter.otlp.jaeger.input]
-	}
-}
 
-otelcol.receiver.otlp "default" {
-	grpc { }
+otelcol.auth.basic "grafanacloud" {
+  username = sys.env("GC_USERNAME")
+  password = sys.env("GC_PASSWORD")
+}
 
-	output {
-		metrics = [otelcol.exporter.debug.console.input]
-		logs    = [otelcol.exporter.debug.console.input]
-		traces  = [otelcol.processor.batch.default.input]
-	}
+otelcol.exporter.otlphttp "grafanacloud" {
+  client {
+    endpoint = sys.env("GC_ENDPOINT")
+    auth     = otelcol.auth.basic.grafanacloud.handler
+  }
 }

config.alloy

@@ -5,6 +5,8 @@ services:
       context: .
       target: dev
     env_file: local.env
+    environment:
+      APP_NAME: "bootstrap-fastapi"
     ports:
       - '8000:8000'
     working_dir: "/app/src"
@@ -18,34 +20,14 @@ services:
       - ./http_app/dev_server.py
 
   dev-socketio:
-    <<: *dev
-    ports:
-      - '8001:8001'
-    command:
-      - python
-      - ./socketio_app/dev_server.py
-
-  otel-http:
     <<: *dev
     environment:
-      OTEL_SERVICE_NAME: "bootstrap-fastapi-dev"
-    command:
-      - opentelemetry-instrument
-      - python
-      - -m
-      - http_app
-
-  otel-socketio:
-    <<: *dev
-    environment:
-      OTEL_SERVICE_NAME: "bootstrap-socketio-dev"
+      APP_NAME: "bootstrap-socketio"
     ports:
       - '8001:8001'
     command:
-      - opentelemetry-instrument
       - python
-      - -m
-      - socketio_app
+      - ./socketio_app/dev_server.py
 
   #########################
   #### Helper services ####
@@ -79,10 +61,9 @@ services:
   dramatiq-worker:
     <<: *dev
     environment:
-      OTEL_SERVICE_NAME: "bootstrap-fastapi-dramatiq-worker"
+      APP_NAME: "bootstrap-dramatiq-worker"
     ports: []
     command:
-      - opentelemetry-instrument
       - dramatiq
       - --watch
       - .

docker-compose.yaml

@@ -2,4 +2,5 @@ ENVIRONMENT: "local"
 AUTH__JWKS_URL: "http://oathkeeper:4456/.well-known/jwks.json"
 #DRAMATIQ__REDIS_URL: "redis://redis:6379/0"
 OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
-OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true"
+OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_REQUEST: ".*"
+OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE: ".*"

local.env

@@ -16,11 +16,11 @@ dependencies = [
     "dramatiq[redis,watch]<2.0.0,>=1.17.1",
     "hiredis<4.0.0,>=3.1.0", # Recommended by dramatiq
     "httpx>=0.23.0",
-    "opentelemetry-distro[otlp]",
-    "opentelemetry-instrumentation",
     "opentelemetry-instrumentation-httpx",
     "opentelemetry-instrumentation-sqlalchemy",
     "opentelemetry-instrumentor-dramatiq",
+    "opentelemetry-exporter-otlp",
+    "opentelemetry-sdk",
     "orjson<4.0.0,>=3.10.12",
     "pydantic<3.0.0,>=2.2.1",
     "pydantic-asyncapi>=0.2.1",
@@ -36,11 +36,12 @@ http = [
     "cryptography>=44.0.0",
     "fastapi>=0.99.0",
     "jinja2<4.0.0,>=3.1.2",
-    # We use the generic ASGI instrumentation, so that if we decide to change
-    # Framework it will still work consistently.
-    "opentelemetry-instrumentation-asgi",
+    # FastAPI instrumentation is based on the generic ASGI instrumentation,
+    # but automatically creates span when routes are invoked.
+    # If we decide to change framework, the generic ASGI instrumentation
+    # will still name metrics with a generic naming.
+    "opentelemetry-instrumentation-fastapi",
     "pyjwt>=2.10.1",
-    "starlette-prometheus<1.0.0,>=0.10.0",
     "strawberry-graphql[debug-server]>=0.204.0",
     "uvicorn[standard]<1.0.0,>=0.34.0",
 ]

pyproject.toml

@@ -10,6 +10,7 @@
 from .dramatiq import init_dramatiq
 from .logs import init_logger
 from .storage import init_storage
+from .telemetry import instrument_opentelemetry
 
 
 class InitReference(BaseModel):
@@ -29,6 +30,7 @@ def application_init(app_config: AppConfig) -> InitReference:
     init_storage()
     init_dramatiq(app_config)
     init_asyncapi_info(app_config.APP_NAME)
+    instrument_opentelemetry(app_config)
 
     return InitReference(
         di_container=container,

src/common/bootstrap.py

@@ -38,3 +38,7 @@ class AppConfig(BaseSettings):
             async_engine=True,
         ),
     )
+    OTEL_EXPORTER_OTLP_ENDPOINT: Optional[str] = None
+    OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: Optional[str] = None
+    OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: Optional[str] = None
+    OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: Optional[str] = None

src/common/config.py

@@ -7,7 +7,6 @@
 from dramatiq.brokers.stub import StubBroker
 from dramatiq.encoder import DecodeError, Encoder, MessageData
 from dramatiq.middleware import AsyncIO
-from opentelemetry_instrumentor_dramatiq import DramatiqInstrumentor
 
 from .config import AppConfig
 
@@ -28,10 +27,6 @@ def decode(self, data: bytes) -> MessageData:
 def init_dramatiq(config: AppConfig):
     broker: Broker
 
-    dramatiq_instrumentor = DramatiqInstrumentor()
-    if not dramatiq_instrumentor.is_instrumented_by_opentelemetry:
-        dramatiq_instrumentor.instrument()
-
     if config.DRAMATIQ.REDIS_URL is not None:
         broker = RedisBroker(url=config.DRAMATIQ.REDIS_URL)
     else: