fix: external traces now respect parent sampling, and prevent broken traces when there is no external trace context (#2395)

* fix: external traces now respect parent sampling, and prevent broken traces when there is no external trace context

* Add changeset

* improve trace flag handling and better internal host checking

* the traceFlags are now being properly passed through as a number

Author: ericallam

.changeset/violet-llamas-roll.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+fix: external traces now respect parent sampling, and prevent broken traces when there is no external trace context

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -408,7 +408,8 @@ export class RunEngineTriggerTaskService {
 
     const newExternalTraceparent = serializeTraceparent(
       parsedTraceparent.traceId,
-      parentSpanId ?? parsedTraceparent.spanId
+      parentSpanId ?? parsedTraceparent.spanId,
+      parsedTraceparent.traceFlags
     );
 
     return {

packages/core/src/v3/isomorphic/traceContext.ts

@@ -1,6 +1,6 @@
 export function parseTraceparent(
   traceparent?: string
-): { traceId: string; spanId: string } | undefined {
+): { traceId: string; spanId: string; traceFlags?: string } | undefined {
   if (!traceparent) {
     return undefined;
   }
@@ -11,7 +11,7 @@ export function parseTraceparent(
     return undefined;
   }
 
-  const [version, traceId, spanId] = parts;
+  const [version, traceId, spanId, traceFlags] = parts;
 
   if (version !== "00") {
     return undefined;
@@ -21,9 +21,9 @@ export function parseTraceparent(
     return undefined;
   }
 
-  return { traceId, spanId };
+  return { traceId, spanId, traceFlags };
 }
 
-export function serializeTraceparent(traceId: string, spanId: string) {
-  return `00-${traceId}-${spanId}-01`;
+export function serializeTraceparent(traceId: string, spanId: string, traceFlags?: string) {
+  return `00-${traceId}-${spanId}-${traceFlags ?? "01"}`;
 }

packages/core/src/v3/otel/tracingSDK.ts

@@ -1,4 +1,10 @@
-import { DiagConsoleLogger, DiagLogLevel, TracerProvider, diag } from "@opentelemetry/api";
+import {
+  DiagConsoleLogger,
+  DiagLogLevel,
+  TraceFlags,
+  TracerProvider,
+  diag,
+} from "@opentelemetry/api";
 import { logs } from "@opentelemetry/api-logs";
 import { TraceState } from "@opentelemetry/core";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
@@ -21,7 +27,7 @@ import {
   SimpleSpanProcessor,
   SpanExporter,
 } from "@opentelemetry/sdk-trace-node";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
+import { SemanticResourceAttributes, SEMATTRS_HTTP_URL } from "@opentelemetry/semantic-conventions";
 import { VERSION } from "../../version.js";
 import {
   OTEL_ATTRIBUTE_PER_EVENT_COUNT_LIMIT,
@@ -287,17 +293,24 @@ function setLogLevel(level: TracingDiagnosticLogLevel) {
 }
 
 class ExternalSpanExporterWrapper {
+  private readonly _isExternallySampled: boolean;
+
   constructor(
     private underlyingExporter: SpanExporter,
     private externalTraceId: string,
     private externalTraceContext:
-      | { traceId: string; spanId: string; tracestate?: string }
+      | { traceId: string; spanId: string; traceFlags: number; tracestate?: string }
       | undefined
-  ) {}
+  ) {
+    this._isExternallySampled = isTraceFlagSampled(externalTraceContext?.traceFlags);
+  }
 
   private transformSpan(span: ReadableSpan): ReadableSpan | undefined {
-    if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) {
-      // Skip partial spans
+    if (!this._isExternallySampled) {
+      return;
+    }
+
+    if (isSpanInternalOnly(span)) {
       return;
     }
 
@@ -325,8 +338,10 @@ class ExternalSpanExporterWrapper {
         traceState: this.externalTraceContext.tracestate
           ? new TraceState(this.externalTraceContext.tracestate)
           : undefined,
-        traceFlags: parentSpanContext?.traceFlags ?? 0,
+        traceFlags: this.externalTraceContext.traceFlags,
       };
+    } else if (isAttemptSpan) {
+      parentSpanContext = undefined;
     }
 
     return {
@@ -360,15 +375,25 @@ class ExternalSpanExporterWrapper {
 }
 
 class ExternalLogRecordExporterWrapper {
+  private readonly _isExternallySampled: boolean;
+
   constructor(
     private underlyingExporter: LogRecordExporter,
     private externalTraceId: string,
     private externalTraceContext:
-      | { traceId: string; spanId: string; tracestate?: string }
+      | { traceId: string; spanId: string; tracestate?: string; traceFlags: number }
       | undefined
-  ) {}
+  ) {
+    this._isExternallySampled = isTraceFlagSampled(externalTraceContext?.traceFlags);
+  }
 
   export(logs: any[], resultCallback: (result: any) => void): void {
+    if (!this._isExternallySampled) {
+      this.underlyingExporter.export([], resultCallback);
+
+      return;
+    }
+
     const modifiedLogs = logs.map(this.transformLogRecord.bind(this));
 
     this.underlyingExporter.export(modifiedLogs, resultCallback);
@@ -410,3 +435,57 @@ class ExternalLogRecordExporterWrapper {
     });
   }
 }
+
+function isSpanInternalOnly(span: ReadableSpan): boolean {
+  if (span.attributes[SemanticInternalAttributes.SPAN_PARTIAL]) {
+    // Skip partial spans
+    return true;
+  }
+
+  const urlPath = span.attributes["url.path"];
+
+  if (typeof urlPath === "string" && urlPath === "/api/v1/usage/ingest") {
+    return true;
+  }
+
+  const httpUrl = span.attributes[SEMATTRS_HTTP_URL] ?? span.attributes["url.full"];
+
+  const url = safeParseUrl(httpUrl);
+
+  if (!url) {
+    return false;
+  }
+
+  const internalHosts = [
+    "api.trigger.dev",
+    "billing.trigger.dev",
+    "cloud.trigger.dev",
+    "engine.trigger.dev",
+    "platform.trigger.dev",
+  ];
+
+  return (
+    internalHosts.some((host) => url.hostname.includes(host)) ||
+    url.pathname.includes("/api/v1/usage/ingest")
+  );
+}
+
+function safeParseUrl(url: unknown): URL | undefined {
+  if (typeof url !== "string") {
+    return undefined;
+  }
+
+  try {
+    return new URL(url);
+  } catch (e) {
+    return undefined;
+  }
+}
+
+function isTraceFlagSampled(traceFlags?: number): boolean {
+  if (typeof traceFlags !== "number") {
+    return true;
+  }
+
+  return (traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED;
+}

packages/core/src/v3/traceContext/manager.ts

@@ -1,5 +1,6 @@
 import { Context, context, propagation, trace, TraceFlags } from "@opentelemetry/api";
 import { TraceContextManager } from "./types.js";
+import { parseTraceParent } from "@opentelemetry/core";
 
 export class StandardTraceContextManager implements TraceContextManager {
   public traceContext: Record<string, unknown> = {};
@@ -39,7 +40,7 @@ export class StandardTraceContextManager implements TraceContextManager {
     const spanContext = {
       traceId: externalTraceContext.traceId,
       spanId: currentSpanContext.spanId,
-      traceFlags: TraceFlags.SAMPLED,
+      traceFlags: externalTraceContext.traceFlags,
       isRemote: true,
     };
 
@@ -60,15 +61,16 @@ function extractExternalTraceContext(traceContext: unknown) {
       : undefined;
 
   if ("traceparent" in traceContext && typeof traceContext.traceparent === "string") {
-    const [version, traceId, spanId] = traceContext.traceparent.split("-");
+    const externalSpanContext = parseTraceParent(traceContext.traceparent);
 
-    if (!traceId || !spanId) {
+    if (!externalSpanContext) {
       return undefined;
     }
 
     return {
-      traceId,
-      spanId,
+      traceId: externalSpanContext.traceId,
+      spanId: externalSpanContext.spanId,
+      traceFlags: externalSpanContext.traceFlags,
       tracestate: tracestate,
     };
   }

packages/core/src/v3/traceContext/types.ts

@@ -8,6 +8,7 @@ export interface TraceContextManager {
     | {
         traceId: string;
         spanId: string;
+        traceFlags: number;
         tracestate?: string;
       }
     | undefined;

pnpm-lock.yaml

@@ -26,6 +26,8 @@
     "@opentelemetry/api-logs": "^0.203.0",
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
+    "@opentelemetry/instrumentation-http": "0.203.0",
+    "@opentelemetry/instrumentation-undici": "0.14.0",
     "@opentelemetry/instrumentation": "^0.203.0",
     "@opentelemetry/sdk-logs": "^0.203.0",
     "@radix-ui/react-avatar": "^1.1.3",

references/d3-chat/package.json

@@ -1,5 +1,5 @@
 import { registerOTel } from "@vercel/otel";
 
 export function register() {
-  registerOTel({ serviceName: "d3-chat" });
+  registerOTel({ serviceName: "d3-chat", traceSampler: "traceidratio" });
 }

references/d3-chat/src/instrumentation.ts

@@ -3,11 +3,14 @@ import { pythonExtension } from "@trigger.dev/python/extension";
 import { installPlaywrightChromium } from "./src/extensions/playwright";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
+import { UndiciInstrumentation } from "@opentelemetry/instrumentation-undici";
 
 export default defineConfig({
   project: process.env.TRIGGER_PROJECT_REF!,
   dirs: ["./src/trigger"],
   telemetry: {
+    instrumentations: [new HttpInstrumentation(), new UndiciInstrumentation()],
     logExporters: [
       new OTLPLogExporter({
         url: "https://api.axiom.co/v1/logs",