chore: PoC + ipynb

Author: rishisurana-labelbox

examples/annotation_import/audio_temporal.ipynb

@@ -19,6 +19,9 @@
 from .video import MaskInstance
 from .video import VideoMaskAnnotation
 
+from .audio import AudioClassificationAnnotation
+from .audio import AudioObjectAnnotation
+
 from .ner import ConversationEntity
 from .ner import DocumentEntity
 from .ner import DocumentTextSelection

libs/labelbox/src/labelbox/data/annotation_types/__init__.py

@@ -0,0 +1,109 @@
+from typing import Optional
+
+from labelbox.data.annotation_types.annotation import ClassificationAnnotation, ObjectAnnotation
+from labelbox.data.mixins import ConfidenceNotSupportedMixin, CustomMetricsNotSupportedMixin
+
+
+class AudioClassificationAnnotation(ClassificationAnnotation):
+    """Audio classification for specific time range
+    
+    Examples:
+    - Speaker identification from 2.5s to 4.1s
+    - Audio quality assessment for a segment
+    - Language detection for audio segments
+    
+    Args:
+        name (Optional[str]): Name of the classification
+        feature_schema_id (Optional[Cuid]): Feature schema identifier
+        value (Union[Text, Checklist, Radio]): Classification value
+        frame (int): The frame index in milliseconds (e.g., 2500 = 2.5 seconds)
+        segment_index (Optional[int]): Index of audio segment this annotation belongs to
+        extra (Dict[str, Any]): Additional metadata
+    """
+
+    frame: int
+    segment_index: Optional[int] = None
+    
+    @classmethod
+    def from_time_range(cls, start_sec: float, end_sec: float, **kwargs):
+        """Create from seconds (user-friendly) to frames (internal)
+        
+        Args:
+            start_sec (float): Start time in seconds
+            end_sec (float): End time in seconds  
+            **kwargs: Additional arguments for the annotation
+            
+        Returns:
+            AudioClassificationAnnotation: Annotation with frame set to start_sec * 1000
+            
+        Example:
+            >>> AudioClassificationAnnotation.from_time_range(
+            ...     start_sec=2.5, end_sec=4.1,
+            ...     name="speaker_id",
+            ...     value=lb_types.Radio(answer=lb_types.ClassificationAnswer(name="john"))
+            ... )
+        """
+        return cls(frame=int(start_sec * 1000), **kwargs)
+    
+    @property
+    def start_time(self) -> float:
+        """Convert frame to seconds for user-facing APIs
+        
+        Returns:
+            float: Time in seconds (e.g., 2500 -> 2.5)
+        """
+        return self.frame / 1000.0
+
+
+class AudioObjectAnnotation(ObjectAnnotation, ConfidenceNotSupportedMixin, CustomMetricsNotSupportedMixin):
+    """Audio object annotation for specific time range
+    
+    Examples:
+    - Transcription: "Hello world" from 2.5s to 4.1s
+    - Sound events: "Dog barking" from 10s to 12s
+    - Audio segments with metadata
+    
+    Args:
+        name (Optional[str]): Name of the annotation
+        feature_schema_id (Optional[Cuid]): Feature schema identifier
+        value (Union[TextEntity, Geometry]): Localization or text content
+        frame (int): The frame index in milliseconds (e.g., 10000 = 10.0 seconds)
+        keyframe (bool): Whether this is a keyframe annotation (default: True)
+        segment_index (Optional[int]): Index of audio segment this annotation belongs to
+        classifications (Optional[List[ClassificationAnnotation]]): Optional sub-classifications
+        extra (Dict[str, Any]): Additional metadata
+    """
+
+    frame: int
+    keyframe: bool = True
+    segment_index: Optional[int] = None
+    
+    @classmethod
+    def from_time_range(cls, start_sec: float, end_sec: float, **kwargs):
+        """Create from seconds (user-friendly) to frames (internal)
+        
+        Args:
+            start_sec (float): Start time in seconds
+            end_sec (float): End time in seconds
+            **kwargs: Additional arguments for the annotation
+            
+        Returns:
+            AudioObjectAnnotation: Annotation with frame set to start_sec * 1000
+            
+        Example:
+            >>> AudioObjectAnnotation.from_time_range(
+            ...     start_sec=10.0, end_sec=12.5,
+            ...     name="transcription",
+            ...     value=lb_types.TextEntity(text="Hello world")
+            ... )
+        """
+        return cls(frame=int(start_sec * 1000), **kwargs)
+    
+    @property
+    def start_time(self) -> float:
+        """Convert frame to seconds for user-facing APIs
+        
+        Returns:
+            float: Time in seconds (e.g., 10000 -> 10.0)
+        """
+        return self.frame / 1000.0

libs/labelbox/src/labelbox/data/annotation_types/audio.py

@@ -13,6 +13,7 @@
 from .metrics import ScalarMetric, ConfusionMatrixMetric
 from .video import VideoClassificationAnnotation
 from .video import VideoObjectAnnotation, VideoMaskAnnotation
+from .audio import AudioClassificationAnnotation, AudioObjectAnnotation
 from .mmc import MessageEvaluationTaskAnnotation
 from pydantic import BaseModel, field_validator
 
@@ -44,6 +45,8 @@ class Label(BaseModel):
             ClassificationAnnotation,
             ObjectAnnotation,
             VideoMaskAnnotation,
+            AudioClassificationAnnotation,
+            AudioObjectAnnotation,
             ScalarMetric,
             ConfusionMatrixMetric,
             RelationshipAnnotation,
@@ -85,6 +88,27 @@ def frame_annotations(
                 frame_dict[annotation.frame].append(annotation)
         return frame_dict
 
+    def audio_annotations_by_frame(
+        self,
+    ) -> Dict[int, List[Union[AudioObjectAnnotation, AudioClassificationAnnotation]]]:
+        """Get audio annotations organized by frame (millisecond)
+        
+        Returns:
+            Dict[int, List]: Dictionary mapping frame (milliseconds) to list of audio annotations
+            
+        Example:
+            >>> label.audio_annotations_by_frame()
+            {2500: [AudioClassificationAnnotation(...)], 10000: [AudioObjectAnnotation(...)]}
+        """
+        frame_dict = defaultdict(list)
+        for annotation in self.annotations:
+            if isinstance(
+                annotation,
+                (AudioObjectAnnotation, AudioClassificationAnnotation),
+            ):
+                frame_dict[annotation.frame].append(annotation)
+        return dict(frame_dict)
+
     def add_url_to_masks(self, signer) -> "Label":
         """
         Creates signed urls for all masks in the Label.

libs/labelbox/src/labelbox/data/annotation_types/label.py

@@ -12,6 +12,7 @@
 
 from ...annotation_types.annotation import ClassificationAnnotation
 from ...annotation_types.video import VideoClassificationAnnotation
+from ...annotation_types.audio import AudioClassificationAnnotation
 from ...annotation_types.llm_prompt_response.prompt import (
     PromptClassificationAnnotation,
     PromptText,
@@ -425,7 +426,7 @@ def to_common(
     def from_common(
         cls,
         annotation: Union[
-            ClassificationAnnotation, VideoClassificationAnnotation
+            ClassificationAnnotation, VideoClassificationAnnotation, AudioClassificationAnnotation
         ],
         data: GenericDataRowData,
     ) -> Union[NDTextSubclass, NDChecklistSubclass, NDRadioSubclass]:
@@ -448,7 +449,7 @@ def from_common(
     @staticmethod
     def lookup_classification(
         annotation: Union[
-            ClassificationAnnotation, VideoClassificationAnnotation
+            ClassificationAnnotation, VideoClassificationAnnotation, AudioClassificationAnnotation
         ],
     ) -> Union[NDText, NDChecklist, NDRadio]:
         return {Text: NDText, Checklist: NDChecklist, Radio: NDRadio}.get(

libs/labelbox/src/labelbox/data/serialization/ndjson/classification.py

@@ -24,6 +24,10 @@
     VideoMaskAnnotation,
     VideoObjectAnnotation,
 )
+from ...annotation_types.audio import (
+    AudioClassificationAnnotation,
+    AudioObjectAnnotation,
+)
 from labelbox.types import DocumentRectangle, DocumentEntity
 from .classification import (
     NDChecklistSubclass,
@@ -69,6 +73,7 @@ def from_common(
             yield from cls._create_relationship_annotations(label)
             yield from cls._create_non_video_annotations(label)
             yield from cls._create_video_annotations(label)
+            yield from cls._create_audio_annotations(label)
 
     @staticmethod
     def _get_consecutive_frames(
@@ -159,6 +164,40 @@ def _create_video_annotations(
                     segments.append(segment)
                 yield NDObject.from_common(segments, label.data)
 
+    @classmethod
+    def _create_audio_annotations(
+        cls, label: Label
+    ) -> Generator[Union[NDChecklistSubclass, NDRadioSubclass], None, None]:
+        """Create audio annotations
+        
+        Args:
+            label: Label containing audio annotations to be processed
+            
+        Yields:
+            NDClassification or NDObject: Audio annotations in NDJSON format
+        """
+        audio_annotations = defaultdict(list)
+        for annot in label.annotations:
+            if isinstance(
+                annot, (AudioClassificationAnnotation, AudioObjectAnnotation)
+            ):
+                audio_annotations[annot.feature_schema_id or annot.name].append(
+                    annot
+                )
+
+        for annotation_group in audio_annotations.values():
+            # For audio, treat each annotation as a single frame (no segments needed)
+            if isinstance(annotation_group[0], AudioClassificationAnnotation):
+                annotation = annotation_group[0]
+                # Add frame information to extra (milliseconds)
+                annotation.extra.update({"frame": annotation.frame})
+                yield NDClassification.from_common(annotation, label.data)
+
+            elif isinstance(annotation_group[0], AudioObjectAnnotation):
+                # For audio objects, treat like single video frame
+                annotation = annotation_group[0]
+                yield NDObject.from_common(annotation, label.data)
+
     @classmethod
     def _create_non_video_annotations(cls, label: Label):
         non_video_annotations = [
@@ -170,6 +209,8 @@ def _create_non_video_annotations(cls, label: Label):
                     VideoClassificationAnnotation,
                     VideoObjectAnnotation,
                     VideoMaskAnnotation,
+                    AudioClassificationAnnotation,
+                    AudioObjectAnnotation,
                     RelationshipAnnotation,
                 ),
             )

libs/labelbox/src/labelbox/data/serialization/ndjson/label.py

@@ -14,6 +14,9 @@
 from labelbox.data.annotation_types.video import (
     VideoObjectAnnotation,
 )
+from labelbox.data.annotation_types.audio import (
+    AudioObjectAnnotation,
+)
 from labelbox.data.mixins import (
     ConfidenceMixin,
     CustomMetric,
@@ -715,6 +718,7 @@ def from_common(
             ObjectAnnotation,
             List[List[VideoObjectAnnotation]],
             VideoMaskAnnotation,
+            AudioObjectAnnotation,
         ],
         data: GenericDataRowData,
     ) -> Union[
@@ -742,6 +746,9 @@ def from_common(
             return obj.from_common(**args)
         elif obj == NDVideoMasks:
             return obj.from_common(annotation, data)
+        elif isinstance(annotation, AudioObjectAnnotation):
+            # Handle audio object annotation like single video frame
+            return cls._handle_single_audio_annotation(annotation, data)
 
         subclasses = [
             NDSubclassification.from_common(annot)
@@ -765,6 +772,41 @@ def from_common(
             **optional_kwargs,
         )
 
+    @classmethod
+    def _handle_single_audio_annotation(cls, annotation: AudioObjectAnnotation, data: GenericDataRowData):
+        """Handle single audio annotation like video frame
+        
+        Args:
+            annotation: Audio object annotation to process
+            data: Data row data
+            
+        Returns:
+            NDObject: Serialized audio object annotation
+        """
+        # Get the appropriate NDObject subclass based on the annotation value type
+        obj = cls.lookup_object(annotation)
+        
+        # Process sub-classifications if any
+        subclasses = [
+            NDSubclassification.from_common(annot)
+            for annot in annotation.classifications
+        ]
+        
+        # Add frame information to extra (milliseconds)
+        extra = annotation.extra.copy() if annotation.extra else {}
+        extra.update({"frame": annotation.frame})
+        
+        # Create the NDObject with frame information
+        return obj.from_common(
+            str(annotation._uuid),
+            annotation.value,
+            subclasses,
+            annotation.name,
+            annotation.feature_schema_id,
+            extra,
+            data,
+        )
+
     @staticmethod
     def lookup_object(
         annotation: Union[ObjectAnnotation, List],