chore: refactor tracer partial flushing logic (#14860)

## Description

This simplify the code and removes the edge case where we tag a complete
trace chunk with unnecessary partial flush tag

We move some of the complexity of removing finished spans from a trace
into a method.

We add __slots__ to _Trace object to aid in memory usage and size.

Small performance win to use `span.duration_is is not None` to check if
a span if finished instead of using the `finished` property function.

## Testing

<!-- Describe your testing strategy or note what tests are included -->

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

Author: brettlangdon

ddtrace/_trace/processor/__init__.py

@@ -263,9 +263,20 @@ def process_trace(self, trace: List[Span]) -> Optional[List[Span]]:
 
 
 class _Trace:
-    def __init__(self, spans=None, num_finished=0):
-        self.spans = spans if spans is not None else []
-        self.num_finished = num_finished
+    __slots__ = ("spans", "num_finished")
+
+    def __init__(self, spans: Optional[List[Span]] = None, num_finished: int = 0):
+        self.spans: List[Span] = spans if spans is not None else []
+        self.num_finished: int = num_finished
+
+    def remove_finished(self) -> List[Span]:
+        # perf: Avoid Span.finished which is a computed property and has function call overhead
+        #       so check Span.duration_ns manually.
+        finished = [s for s in self.spans if s.duration_ns is not None]
+        if finished:
+            self.spans[:] = [s for s in self.spans if s.duration_ns is None]
+            self.num_finished = 0
+        return finished
 
 
 class SpanAggregator(SpanProcessor):
@@ -350,31 +361,22 @@ def on_span_finish(self, span: Span) -> None:
                 return
 
             trace = self._traces[span.trace_id]
-            num_buffered = len(trace.spans)
             trace.num_finished += 1
-            should_partial_flush = self.partial_flush_enabled and trace.num_finished >= self.partial_flush_min_spans
-            is_trace_complete = trace.num_finished >= len(trace.spans)
-            if not is_trace_complete and not should_partial_flush:
-                return
-
-            if not is_trace_complete:
-                finished = [s for s in trace.spans if s.finished]
-                if not finished:
-                    return
-                trace.spans[:] = [s for s in trace.spans if not s.finished]  # In-place update
-                trace.num_finished = 0
-            else:
+            num_buffered = len(trace.spans)
+            is_trace_complete = trace.num_finished >= num_buffered
+            num_finished = trace.num_finished
+            should_partial_flush = False
+            if is_trace_complete:
                 finished = trace.spans
                 del self._traces[span.trace_id]
                 # perf: Flush span finish metrics to the telemetry writer after the trace is complete
                 self._queue_span_count_metrics("spans_finished", "integration_name")
-
-        num_finished = len(finished)
-        if should_partial_flush:
-            # FIXME(munir): should_partial_flush should return false if all the spans in the trace are finished.
-            # For example if partial flushing min spans is 10 and the trace has 10 spans, the trace should
-            # not have a partial flush metric. This trace was processed in its entirety.
-            finished[0].set_metric("_dd.py.partial_flush", num_finished)
+            elif self.partial_flush_enabled and num_finished >= self.partial_flush_min_spans:
+                should_partial_flush = True
+                finished = trace.remove_finished()
+                finished[0].set_metric("_dd.py.partial_flush", num_finished)
+            else:
+                return
 
         # perf: Process spans outside of the span aggregator lock
         spans = finished

tests/snapshots/test_selenium_chrome.test_selenium_chrome_pytest_rum_disabled.json

@@ -33,7 +33,6 @@
       "type": "test_session_end"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,
@@ -174,7 +173,6 @@
       "type": "test"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,

tests/snapshots/test_selenium_chrome.test_selenium_chrome_pytest_rum_enabled.json

@@ -33,7 +33,6 @@
       "type": "test_session_end"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,
@@ -175,7 +174,6 @@
       "type": "test"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,

tests/snapshots/test_selenium_chrome.test_selenium_chrome_pytest_unpatch_does_not_record_selenium_tags.json

@@ -33,7 +33,6 @@
       "type": "test_session_end"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,
@@ -169,7 +168,6 @@
       "type": "test"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,

tests/snapshots/tests.contrib.pytest.test_pytest_snapshot_v2.test_pytest_will_include_lines_pct.json

@@ -11,43 +11,42 @@
     "meta": {
       "_dd.origin": "ciapp-test",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "6751748100000000",
+      "_dd.p.tid": "68ee770a00000000",
       "component": "pytest",
       "language": "python",
-      "library_version": "2.18.0.dev124+gc03b9e422",
-      "os.architecture": "x86_64",
+      "library_version": "3.17.0.dev37+g6f7c158e6.d20251014",
+      "os.architecture": "aarch64",
       "os.platform": "Linux",
-      "os.version": "5.15.0-1071-aws",
-      "runtime-id": "5714e89371d3405f970eb00f53936a67",
+      "os.version": "6.10.14-linuxkit",
+      "runtime-id": "cb2348822a0142f7923918d45063d5d0",
       "runtime.name": "CPython",
-      "runtime.version": "3.10.14",
+      "runtime.version": "3.12.11",
       "span.kind": "test",
       "test.command": "pytest --ddtrace",
       "test.framework": "pytest",
-      "test.framework_version": "6.2.5",
+      "test.framework_version": "8.4.2",
       "test.module": "",
       "test.module_path": "",
       "test.name": "test_add_two_number_list",
       "test.source.file": "test_tools.py",
       "test.status": "pass",
       "test.suite": "test_tools.py",
       "test.type": "test",
-      "test_module_id": "18268695399323445148",
-      "test_session_id": "15721560387983696792",
-      "test_suite_id": "5983435943999745407",
+      "test_module_id": "5787288656364304641",
+      "test_session_id": "5197474445235026729",
+      "test_suite_id": "8497613065864877119",
       "type": "test"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,
-      "process_id": 35337,
+      "process_id": 4189,
       "test.source.end": 9,
       "test.source.start": 3
     },
-    "duration": 2404850,
-    "start": 1733391489340338256
+    "duration": 1569042,
+    "start": 1760458506005108126
   }],
 [
   {
@@ -62,35 +61,34 @@
     "meta": {
       "_dd.origin": "ciapp-test",
       "_dd.p.dm": "-0",
-      "_dd.p.tid": "6751748100000000",
+      "_dd.p.tid": "68ee770900000000",
       "component": "pytest",
       "language": "python",
-      "library_version": "2.18.0.dev124+gc03b9e422",
-      "os.architecture": "x86_64",
+      "library_version": "3.17.0.dev37+g6f7c158e6.d20251014",
+      "os.architecture": "aarch64",
       "os.platform": "Linux",
-      "os.version": "5.15.0-1071-aws",
-      "runtime-id": "5714e89371d3405f970eb00f53936a67",
+      "os.version": "6.10.14-linuxkit",
+      "runtime-id": "cb2348822a0142f7923918d45063d5d0",
       "runtime.name": "CPython",
-      "runtime.version": "3.10.14",
+      "runtime.version": "3.12.11",
       "span.kind": "test",
       "test.code_coverage.enabled": "false",
       "test.command": "pytest --ddtrace",
       "test.framework": "pytest",
-      "test.framework_version": "6.2.5",
+      "test.framework_version": "8.4.2",
       "test.itr.tests_skipping.enabled": "false",
       "test.status": "pass",
-      "test_session_id": "15721560387983696792",
+      "test_session_id": "5197474445235026729",
       "type": "test_session_end"
     },
     "metrics": {
-      "_dd.py.partial_flush": 1,
       "_dd.top_level": 1,
       "_dd.tracer_kr": 1.0,
       "_sampling_priority_v1": 1,
-      "process_id": 35337
+      "process_id": 4189
     },
-    "duration": 45677337,
-    "start": 1733391489299231481
+    "duration": 30965084,
+    "start": 1760458505977856084
   },
      {
        "name": "pytest.test_module",
@@ -104,35 +102,35 @@
        "meta": {
          "_dd.origin": "ciapp-test",
          "_dd.p.dm": "-0",
-         "_dd.p.tid": "6751748100000000",
+         "_dd.p.tid": "68ee770900000000",
          "component": "pytest",
          "language": "python",
-         "library_version": "2.18.0.dev124+gc03b9e422",
-         "os.architecture": "x86_64",
+         "library_version": "3.17.0.dev37+g6f7c158e6.d20251014",
+         "os.architecture": "aarch64",
          "os.platform": "Linux",
-         "os.version": "5.15.0-1071-aws",
+         "os.version": "6.10.14-linuxkit",
          "runtime.name": "CPython",
-         "runtime.version": "3.10.14",
+         "runtime.version": "3.12.11",
          "span.kind": "test",
          "test.code_coverage.enabled": "false",
          "test.command": "pytest --ddtrace",
          "test.framework": "pytest",
-         "test.framework_version": "6.2.5",
+         "test.framework_version": "8.4.2",
          "test.itr.tests_skipping.enabled": "false",
          "test.module": "",
          "test.module_path": "",
          "test.status": "pass",
-         "test_module_id": "18268695399323445148",
-         "test_session_id": "15721560387983696792",
+         "test_module_id": "5787288656364304641",
+         "test_session_id": "5197474445235026729",
          "type": "test_module_end"
        },
        "metrics": {
          "_dd.py.partial_flush": 1,
          "_dd.tracer_kr": 1.0,
          "_sampling_priority_v1": 1
        },
-       "duration": 4050622,
-       "start": 1733391489340123850
+       "duration": 2606333,
+       "start": 1760458506004979543
      },
         {
           "name": "pytest.test_suite",
@@ -146,33 +144,33 @@
           "meta": {
             "_dd.origin": "ciapp-test",
             "_dd.p.dm": "-0",
-            "_dd.p.tid": "6751748100000000",
+            "_dd.p.tid": "68ee770900000000",
             "component": "pytest",
             "language": "python",
-            "library_version": "2.18.0.dev124+gc03b9e422",
-            "os.architecture": "x86_64",
+            "library_version": "3.17.0.dev37+g6f7c158e6.d20251014",
+            "os.architecture": "aarch64",
             "os.platform": "Linux",
-            "os.version": "5.15.0-1071-aws",
+            "os.version": "6.10.14-linuxkit",
             "runtime.name": "CPython",
-            "runtime.version": "3.10.14",
+            "runtime.version": "3.12.11",
             "span.kind": "test",
             "test.command": "pytest --ddtrace",
             "test.framework": "pytest",
-            "test.framework_version": "6.2.5",
+            "test.framework_version": "8.4.2",
             "test.module": "",
             "test.module_path": "",
             "test.status": "pass",
             "test.suite": "test_tools.py",
-            "test_module_id": "18268695399323445148",
-            "test_session_id": "15721560387983696792",
-            "test_suite_id": "5983435943999745407",
+            "test_module_id": "5787288656364304641",
+            "test_session_id": "5197474445235026729",
+            "test_suite_id": "8497613065864877119",
             "type": "test_suite_end"
           },
           "metrics": {
             "_dd.py.partial_flush": 1,
             "_dd.tracer_kr": 1.0,
             "_sampling_priority_v1": 1
           },
-          "duration": 3652661,
-          "start": 1733391489340244453
+          "duration": 2406917,
+          "start": 1760458506005054584
         }]]