From f59041ea74c9c814c2a60fe0ddc8a8e6de0830f0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 11 Nov 2025 15:10:45 -0500 Subject: [PATCH 1/2] Attempting to reduce contention for virtual threads --- .../java/datadog/trace/core/PendingTrace.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java b/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java index b4b0a6f93ba..2134297adca 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java @@ -234,7 +234,7 @@ PublishState onPublish(final DDSpan span) { if (span == rootSpan) { tracer.onRootSpanPublished(rootSpan); } - return decrementRefAndMaybeWrite(span == rootSpan); + return decrementRefAndMaybeWrite(span == rootSpan, true); } @Override @@ -265,10 +265,10 @@ public void registerContinuation(final AgentScope.Continuation continuation) { @Override public void removeContinuation(final AgentScope.Continuation continuation) { - decrementRefAndMaybeWrite(false); + decrementRefAndMaybeWrite(false, false); } - private PublishState decrementRefAndMaybeWrite(boolean isRootSpan) { + private PublishState decrementRefAndMaybeWrite(boolean isRootSpan, boolean addedSpan) { final int count = PENDING_REFERENCE_COUNT.decrementAndGet(this); if (strictTraceWrites && count < 0) { throw new IllegalStateException("Pending reference count " + count + " is negative"); @@ -283,8 +283,19 @@ private PublishState decrementRefAndMaybeWrite(boolean isRootSpan) { // Finished root with pending work ... delay write pendingTraceBuffer.enqueue(this); return PublishState.ROOT_BUFFERED; - } else if (partialFlushMinSpans > 0 && size() >= partialFlushMinSpans) { + } else if (addedSpan && partialFlushMinSpans > 0 && size() >= partialFlushMinSpans) { // Trace is getting too big, write anything completed. + + // DQH - We only trigger a partial flush, when a span has just been added + // This prevents a bunch of threads which are only performing scope/context operations + // from all fighting to perform the partialFlush after the threshold is crossed. + + // This is an important optimization for virtual threads where a continuation might + // be created even though no span is created. In that situation, virtual threads + // can end up fighting to perform the partialFlush. And even trying to perform a + // partialFlush requires taking the PendingTrace lock which can lead to unmounting + // the virtual thread from its carrier thread. + partialFlush(); return PublishState.PARTIAL_FLUSH; } else if (rootSpanWritten) { From 34a6e6ba5aab28c24a1a7bb51b6b52b8c89268f4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 11 Nov 2025 15:13:50 -0500 Subject: [PATCH 2/2] spotless --- .../java/datadog/trace/core/PendingTrace.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java b/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java index 2134297adca..82177b58fb6 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/PendingTrace.java @@ -285,17 +285,17 @@ private PublishState decrementRefAndMaybeWrite(boolean isRootSpan, boolean added return PublishState.ROOT_BUFFERED; } else if (addedSpan && partialFlushMinSpans > 0 && size() >= partialFlushMinSpans) { // Trace is getting too big, write anything completed. - + // DQH - We only trigger a partial flush, when a span has just been added - // This prevents a bunch of threads which are only performing scope/context operations + // This prevents a bunch of threads which are only performing scope/context operations // from all fighting to perform the partialFlush after the threshold is crossed. - - // This is an important optimization for virtual threads where a continuation might - // be created even though no span is created. In that situation, virtual threads - // can end up fighting to perform the partialFlush. And even trying to perform a - // partialFlush requires taking the PendingTrace lock which can lead to unmounting + + // This is an important optimization for virtual threads where a continuation might + // be created even though no span is created. In that situation, virtual threads + // can end up fighting to perform the partialFlush. And even trying to perform a + // partialFlush requires taking the PendingTrace lock which can lead to unmounting // the virtual thread from its carrier thread. - + partialFlush(); return PublishState.PARTIAL_FLUSH; } else if (rootSpanWritten) {