chore: final nail

Author: rishisurana-labelbox

libs/labelbox/src/labelbox/data/serialization/ndjson/label.py

@@ -76,7 +76,7 @@ def from_common(
             yield from cls._create_relationship_annotations(label)
             yield from cls._create_non_video_annotations(label)
             yield from cls._create_video_annotations(label)
-            yield from cls._create_audio_annotations(label)
+            yield from cls._create_temporal_annotations(label)
 
     @staticmethod
     def _get_consecutive_frames(
@@ -168,7 +168,7 @@ def _create_video_annotations(
                 yield NDObject.from_common(segments, label.data)
 
     @classmethod
-    def _create_audio_annotations(
+    def _create_temporal_annotations(
         cls, label: Label
     ) -> Generator[BaseModel, None, None]:
         """Create temporal annotations with nested classifications using new temporal classes."""

libs/labelbox/src/labelbox/data/serialization/ndjson/temporal.py

@@ -89,9 +89,14 @@ def _process_text_group(
 
     Each annotation can have multiple (start, end, text) tuples.
     Groups by text value and merges frames.
+
+    Nested classifications are assigned to text values based on frame overlap.
     """
     # Collect all text values with their frames
-    text_data = defaultdict(lambda: {"frames": [], "nested": []})
+    text_data = defaultdict(lambda: {"frames": []})
+
+    # Collect all nested classifications from all annotations
+    all_nested_classifications = []
 
     for ann in annotations:
         for start, end, text_value in ann.value:
@@ -104,9 +109,12 @@ def _process_text_group(
 
             text_data[text_value]["frames"].append({"start": start, "end": end})
 
-            # Collect nested classifications
-            if ann.classifications:
-                text_data[text_value]["nested"].extend(ann.classifications)
+        # Collect nested classifications at annotation level (not per text value)
+        if ann.classifications:
+            all_nested_classifications.extend(ann.classifications)
+
+    # Track which nested classifications were assigned
+    assigned_nested = set()
 
     # Build results
     results = []
@@ -119,15 +127,40 @@ def _process_text_group(
             "frames": unique_frames,
         }
 
-        # Process nested classifications recursively
-        if data["nested"]:
+        # Assign nested classifications based on frame overlap
+        if all_nested_classifications:
             parent_frame_tuples = [(f["start"], f["end"]) for f in unique_frames]
-            nested = _process_nested_classifications(data["nested"], parent_frame_tuples)
-            if nested:
-                entry["classifications"] = nested
+            # Filter nested classifications that overlap with this text value's frames
+            relevant_nested = _filter_classifications_by_overlap(
+                all_nested_classifications, parent_frame_tuples
+            )
+            if relevant_nested:
+                # Track that these were assigned
+                for cls in relevant_nested:
+                    assigned_nested.add(id(cls))
+
+                # Pass ONLY THIS text value's frames so nested answers are filtered correctly
+                nested = _process_nested_classifications(relevant_nested, parent_frame_tuples)
+                if nested:
+                    entry["classifications"] = nested
 
         results.append(entry)
 
+    # Log orphaned nested classifications (not assigned to any parent)
+    if all_nested_classifications:
+        for cls in all_nested_classifications:
+            if id(cls) not in assigned_nested:
+                if isinstance(cls, TemporalClassificationText):
+                    frames_info = cls.value[0][:2] if cls.value else "no frames"
+                elif isinstance(cls, TemporalClassificationQuestion):
+                    frames_info = cls.value[0].frames if cls.value and cls.value[0].frames else "no frames"
+                else:
+                    frames_info = "unknown"
+                logger.warning(
+                    f"Orphaned nested classification '{cls.name}' with frames {frames_info} - "
+                    f"no parent text value found with overlapping frames."
+                )
+
     return results
 
 
@@ -140,28 +173,43 @@ def _process_question_group(
 
     Each annotation has a list of TemporalClassificationAnswer objects.
     Groups by answer name and merges frames.
+
+    Nested classifications are assigned to answers based on frame overlap.
     """
-    # Collect all answers
-    answer_data = defaultdict(lambda: {"frames": [], "nested": []})
+    # Collect all answers with their frames
+    answer_data = defaultdict(lambda: {"frames": []})
+
+    # Collect all nested classifications from all answers
+    all_nested_by_answer = defaultdict(list)
 
     for ann in annotations:
         for answer in ann.value:  # value contains list of answers
             # Validate and collect frames
             valid_frames = []
             for start, end in answer.frames:
-                if parent_frames and not _is_frame_subset([(start, end)], parent_frames):
-                    logger.warning(
-                        f"Answer '{answer.name}' frames ({start}, {end}) not subset of parent frames {parent_frames}. Discarding."
-                    )
-                    continue
+                # If parent_frames provided, check if answer frames are subset of ANY parent frame
+                # A child frame is a subset if: parent_start <= child_start AND child_end <= parent_end
+                if parent_frames:
+                    is_valid = False
+                    for parent_start, parent_end in parent_frames:
+                        if parent_start <= start and end <= parent_end:
+                            is_valid = True
+                            break
+                    if not is_valid:
+                        # Don't log here - this is expected when processing inductive structures
+                        # Only log orphaned classifications that are never assigned to any parent
+                        continue
                 valid_frames.append({"start": start, "end": end})
 
             if valid_frames:  # Only add if we have valid frames
                 answer_data[answer.name]["frames"].extend(valid_frames)
 
-                # Collect nested classifications
+                # Collect nested classifications at answer level
                 if answer.classifications:
-                    answer_data[answer.name]["nested"].extend(answer.classifications)
+                    all_nested_by_answer[answer.name].extend(answer.classifications)
+
+    # Track which nested classifications were assigned
+    assigned_nested = set()
 
     # Build results
     results = []
@@ -177,15 +225,39 @@ def _process_question_group(
             "frames": unique_frames,
         }
 
-        # Process nested classifications recursively
-        if data["nested"]:
+        # Assign nested classifications based on frame overlap
+        if all_nested_by_answer[answer_name]:
             parent_frame_tuples = [(f["start"], f["end"]) for f in unique_frames]
-            nested = _process_nested_classifications(data["nested"], parent_frame_tuples)
-            if nested:
-                entry["classifications"] = nested
+            # Filter nested classifications that overlap with this answer's frames
+            relevant_nested = _filter_classifications_by_overlap(
+                all_nested_by_answer[answer_name], parent_frame_tuples
+            )
+            if relevant_nested:
+                # Track that these were assigned
+                for cls in relevant_nested:
+                    assigned_nested.add(id(cls))
+
+                nested = _process_nested_classifications(relevant_nested, parent_frame_tuples)
+                if nested:
+                    entry["classifications"] = nested
 
         results.append(entry)
 
+    # Log orphaned nested classifications (not assigned to any answer)
+    for answer_name, nested_list in all_nested_by_answer.items():
+        for cls in nested_list:
+            if id(cls) not in assigned_nested:
+                if isinstance(cls, TemporalClassificationText):
+                    frames_info = cls.value[0][:2] if cls.value else "no frames"
+                elif isinstance(cls, TemporalClassificationQuestion):
+                    frames_info = cls.value[0].frames if cls.value and cls.value[0].frames else "no frames"
+                else:
+                    frames_info = "unknown"
+                logger.warning(
+                    f"Orphaned nested classification '{cls.name}' in answer '{answer_name}' with frames {frames_info} - "
+                    f"no overlapping frames found with parent answer."
+                )
+
     return results
 
 
@@ -229,6 +301,57 @@ def _process_nested_classifications(
     return results
 
 
+def _filter_classifications_by_overlap(
+    classifications: List[Union[TemporalClassificationText, TemporalClassificationQuestion]],
+    parent_frames: List[Tuple[int, int]],
+) -> List[Union[TemporalClassificationText, TemporalClassificationQuestion]]:
+    """
+    Filter classifications to only include those with frames that overlap with parent frames.
+
+    A classification is included if ANY of its frame ranges overlap with ANY parent frame range.
+    """
+    relevant = []
+
+    for cls in classifications:
+        has_overlap = False
+
+        # Check frames based on classification type
+        if isinstance(cls, TemporalClassificationText):
+            # Check text value frames
+            for start, end, _ in cls.value:
+                if _frames_overlap([(start, end)], parent_frames):
+                    has_overlap = True
+                    break
+        elif isinstance(cls, TemporalClassificationQuestion):
+            # Check answer frames
+            for answer in cls.value:
+                if _frames_overlap(answer.frames, parent_frames):
+                    has_overlap = True
+                    break
+
+        if has_overlap:
+            relevant.append(cls)
+
+    return relevant
+
+
+def _frames_overlap(
+    frames1: List[Tuple[int, int]],
+    frames2: List[Tuple[int, int]],
+) -> bool:
+    """
+    Check if any frame in frames1 overlaps with any frame in frames2.
+
+    Two frames (s1, e1) and (s2, e2) overlap if:
+    max(s1, s2) <= min(e1, e2)
+    """
+    for start1, end1 in frames1:
+        for start2, end2 in frames2:
+            if max(start1, start2) <= min(end1, end2):
+                return True
+    return False
+
+
 def _is_frame_subset(
     child_frames: List[Tuple[int, int]],
     parent_frames: List[Tuple[int, int]],