Add a _GenericDataType and allow pass a simple dict to Label

Author: Val Brodsky

labelbox/data/annotation_types/data/generic_data_row_data.py

@@ -0,0 +1,28 @@
+from typing import Callable, Literal, Optional
+
+from labelbox import pydantic_compat
+from labelbox.data.annotation_types.data.base_data import BaseData
+from labelbox.utils import _NoCoercionMixin
+
+
+class GenericDataRowData(BaseData, _NoCoercionMixin):
+    """Generic data row data
+    """
+    url: Optional[str] = None
+    class_name: Literal["GenericDataRowData"] = "GenericDataRowData"
+
+    def create_url(self, signer: Callable[[bytes], str]) -> None:
+        return None
+
+    @pydantic_compat.root_validator(pre=True)
+    def validate_one_datarow_key_present(cls, data):
+        keys = ['external_id', 'global_key', 'uid']
+        count = 0
+        for key in keys:
+            if data.get(key):
+                count += 1
+        if count < 1:
+            raise ValueError(f"Exactly one of {keys} must be present.")
+        if count > 1:
+            raise ValueError(f"Only one of {keys} can be present.")
+        return data

labelbox/data/annotation_types/label.py

@@ -5,12 +5,13 @@
 from labelbox import pydantic_compat
 
 import labelbox
+from labelbox.data.annotation_types.data.generic_data_row_data import GenericDataRowData
 from labelbox.data.annotation_types.data.tiled_image import TiledImageData
 from labelbox.schema import ontology
 from .annotation import ClassificationAnnotation, ObjectAnnotation
 from .relationship import RelationshipAnnotation
 from .classification import ClassificationAnswer
-from .data import AudioData, ConversationData, DicomData, DocumentData, HTMLData, ImageData, MaskData, TextData, VideoData, LlmPromptCreationData, LlmPromptResponseCreationData, LlmResponseCreationData
+from .data import AudioData, ConversationData, DicomData, DocumentData, HTMLData, ImageData, TextData, VideoData, LlmPromptCreationData, LlmPromptResponseCreationData, LlmResponseCreationData
 from .geometry import Mask
 from .metrics import ScalarMetric, ConfusionMatrixMetric
 from .types import Cuid
@@ -21,7 +22,7 @@
 DataType = Union[VideoData, ImageData, TextData, TiledImageData, AudioData,
                  ConversationData, DicomData, DocumentData, HTMLData,
                  LlmPromptCreationData, LlmPromptResponseCreationData,
-                 LlmResponseCreationData]
+                 LlmResponseCreationData, GenericDataRowData]
 
 
 class Label(pydantic_compat.BaseModel):
@@ -51,6 +52,18 @@ class Label(pydantic_compat.BaseModel):
                             RelationshipAnnotation]] = []
     extra: Dict[str, Any] = {}
 
+    @staticmethod
+    def is_data_type(data: Union[Dict[str, Any], DataType]) -> bool:
+        if isinstance(data, DataType):
+            return True
+        return False
+
+    @pydantic_compat.root_validator(pre=True)
+    def validate_data(cls, label):
+        if not Label.is_data_type(label.get("data")):
+            label["data"]["class_name"] = "GenericDataRowData"
+        return label
+
     def object_annotations(self) -> List[ObjectAnnotation]:
         return self._get_annotations_by_type(ObjectAnnotation)
 

tests/data/annotation_import/conftest.py

@@ -1885,3 +1885,57 @@ def bbox_video_annotation_objects():
     ]
 
     return bbox_annotation
+
+
+class Helpers:
+
+    @staticmethod
+    def remove_keys_recursive(d, keys):
+        for k in keys:
+            if k in d:
+                del d[k]
+        for k, v in d.items():
+            if isinstance(v, dict):
+                Helpers.remove_keys_recursive(v, keys)
+            elif isinstance(v, list):
+                for i in v:
+                    if isinstance(i, dict):
+                        Helpers.remove_keys_recursive(i, keys)
+
+    @staticmethod
+    # NOTE this uses quite a primitive check for cuids but I do not think it is worth coming up with a better one
+    # Also this function is NOT written with performance in mind, good for small to mid size dicts like we have in our test
+    def rename_cuid_key_recursive(d):
+        new_key = "<cuid>"
+        for k in list(d.keys()):
+            if len(k) == 25 and not k.isalpha():  # primitive check for cuid
+                d[new_key] = d.pop(k)
+        for k, v in d.items():
+            if isinstance(v, dict):
+                Helpers.rename_cuid_key_recursive(v)
+            elif isinstance(v, list):
+                for i in v:
+                    if isinstance(i, dict):
+                        Helpers.rename_cuid_key_recursive(i)
+
+
+@pytest.fixture
+def helpers():
+    return Helpers
+
+
+@pytest.fixture
+def create_data_row_for_project(project, dataset, data_row_ndjson, batch_name):
+    data_row = dataset.create_data_row(data_row_ndjson)
+
+    project.create_batch(
+        batch_name,
+        [data_row.uid],  # sample of data row objects
+        5,  # priority between 1(Highest) - 5(lowest)
+    )
+    project.data_row_ids.append(data_row.uid)
+
+    yield data_row
+
+    data_row.delete()
+    project.delete()

tests/data/annotation_import/test_data_types.py

@@ -5,7 +5,6 @@
 
 import labelbox as lb
 from labelbox.data.annotation_types.data.video import VideoData
-from labelbox.schema.data_row import DataRow
 from labelbox.schema.media_type import MediaType
 import labelbox.types as lb_types
 from labelbox.data.annotation_types.data import (
@@ -70,35 +69,6 @@
 ]
 
 
-def remove_keys_recursive(d, keys):
-    for k in keys:
-        if k in d:
-            del d[k]
-    for k, v in d.items():
-        if isinstance(v, dict):
-            remove_keys_recursive(v, keys)
-        elif isinstance(v, list):
-            for i in v:
-                if isinstance(i, dict):
-                    remove_keys_recursive(i, keys)
-
-
-# NOTE this uses quite a primitive check for cuids but I do not think it is worth coming up with a better one
-# Also this function is NOT written with performance in mind, good for small to mid size dicts like we have in our test
-def rename_cuid_key_recursive(d):
-    new_key = "<cuid>"
-    for k in list(d.keys()):
-        if len(k) == 25 and not k.isalpha():  # primitive check for cuid
-            d[new_key] = d.pop(k)
-    for k, v in d.items():
-        if isinstance(v, dict):
-            rename_cuid_key_recursive(v)
-        elif isinstance(v, list):
-            for i in v:
-                if isinstance(i, dict):
-                    rename_cuid_key_recursive(i)
-
-
 def get_annotation_comparison_dicts_from_labels(labels):
     labels_ndjson = list(NDJsonConverter.serialize(labels))
     for annotation in labels_ndjson:
@@ -161,19 +131,6 @@ def get_annotation_comparison_dicts_from_export(export_result, data_row_id,
     return converted_annotations
 
 
-def create_data_row_for_project(project, dataset, data_row_ndjson, batch_name):
-    data_row = dataset.create_data_row(data_row_ndjson)
-
-    project.create_batch(
-        batch_name,
-        [data_row.uid],  # sample of data row objects
-        5,  # priority between 1(Highest) - 5(lowest)
-    )
-    project.data_row_ids.append(data_row.uid)
-
-    return data_row
-
-
 # TODO: Add VideoData. Currently label import job finishes without errors but project.export_labels() returns empty list.
 @pytest.mark.parametrize(
     "data_type_class",
@@ -190,15 +147,10 @@ def create_data_row_for_project(project, dataset, data_row_ndjson, batch_name):
         LlmResponseCreationData,
     ],
 )
-def test_import_data_types(
-    client,
-    configured_project,
-    initial_dataset,
-    rand_gen,
-    data_row_json_by_data_type,
-    annotations_by_data_type,
-    data_type_class,
-):
+def test_import_data_types(client, configured_project, initial_dataset,
+                           rand_gen, data_row_json_by_data_type,
+                           annotations_by_data_type, data_type_class,
+                           create_data_row_for_project):
     project = configured_project
     project_id = project.uid
     dataset = initial_dataset
@@ -241,6 +193,7 @@ def test_import_data_types_by_global_key(
     rand_gen,
     data_row_json_by_data_type,
     annotations_by_data_type,
+    create_data_row_for_project,
 ):
     project = configured_project
     project_id = project.uid
@@ -331,6 +284,8 @@ def test_import_data_types_v2(
     exports_v2_by_data_type,
     export_v2_test_helpers,
     rand_gen,
+    helpers,
+    create_data_row_for_project,
 ):
     project = configured_project
     dataset = initial_dataset
@@ -381,9 +336,9 @@ def test_import_data_types_v2(
     exported_project_labels = exported_project["labels"][0]
     exported_annotations = exported_project_labels["annotations"]
 
-    remove_keys_recursive(exported_annotations,
-                          ["feature_id", "feature_schema_id"])
-    rename_cuid_key_recursive(exported_annotations)
+    helpers.remove_keys_recursive(exported_annotations,
+                                  ["feature_id", "feature_schema_id"])
+    helpers.rename_cuid_key_recursive(exported_annotations)
     assert exported_annotations == exports_v2_by_data_type[data_type_string]
 
     data_row = client.get_data_row(data_row.uid)
@@ -400,6 +355,7 @@ def test_import_label_annotations(
     data_class,
     annotations,
     rand_gen,
+    create_data_row_for_project,
 ):
     project = configured_project_with_one_data_row
     dataset = initial_dataset