Release 3.59.0 (#1353)

Author: vbrodsky

CHANGELOG.md

@@ -1,4 +1,16 @@
 # Changelog
+# Version 3.59.0 (2024-01-05)
+## Added
+* Support set_labeling_parameter_overrides for global keys 
+* Support bulk_delete of data row metadata for global keys
+* Support bulk_export of data row metadata for global keys
+## Fixed
+* Stop overwriting class annotations on prediction upload
+* Prevent users from uploading video annotations over the API limit (5000)
+* Make description optional for foundry app
+## Notebooks
+* Update notebooks for Project set_labeling_parameter_overrides add support for global keys
+
 # Version 3.58.1 (2023-12-15)
 ## Added
 * Support to export all projects and all model runs to `export_v2` for a `dataset` and a `slice`

docs/source/conf.py

@@ -21,7 +21,7 @@
 copyright = '2021, Labelbox'
 author = 'Labelbox'
 
-release = '3.58.1'
+release = '3.59.0'
 
 # -- General configuration ---------------------------------------------------
 

docs/source/index.rst

@@ -179,6 +179,7 @@ DataRowMetadata
 
 .. automodule:: labelbox.schema.data_row_metadata
    :members:
+   :exclude-members: _DeleteBatchDataRowMetadata
    :show-inheritance:
 
 AnnotationImport
@@ -236,6 +237,12 @@ Identifiables
    :members:
    :show-inheritance:
 
+Identifiable
+---------------------------
+.. automodule:: labelbox.schema.identifiable
+   :members:
+   :show-inheritance:
+
 ConflictResolutionStrategy
 ---------------------------
 .. automodule:: labelbox.schema.conflict_resolution_strategy

examples/project_configuration/queue_management.ipynb

@@ -73,14 +73,14 @@
       "metadata": {},
       "source": [
         "# API Key and Client\n",
-        "Provide a valid api key below in order to properly connect to the Labelbox Client."
+        "See the developer guide for [creating an API key](https://docs.labelbox.com/reference/create-api-key)."
       ],
       "cell_type": "markdown"
     },
     {
       "metadata": {},
       "source": [
-        "# Add your api key\n",
+        "# Add your API key\n",
         "API_KEY = \"\"\n",
         "client = lb.Client(api_key=API_KEY)"
       ],
@@ -98,9 +98,7 @@
     {
       "metadata": {},
       "source": [
-        "## Create two Labelbox  projects\n",
-        "# Project defaults to batch mode with benchmark quality settings if queue mode argument is not provided\n",
-        "# Note that queue mode will be deprecated once dataset mode is deprecated \n",
+        "# Create Labelbox  project\n",
         "\n",
         "project = client.create_project(name=\"batch-test-project\",\n",
         "                                description=\"a description\",\n",
@@ -139,7 +137,7 @@
         "data_rows = dataset.create_data_rows(uploads)\n",
         "data_rows.wait_till_done()\n",
         "print(\"Errors\" , data_rows.errors)\n",
-        "print(\"Dataset status: \", data_rows.status)"
+        "print(\"Dataset status: \", data_rows.status)\n"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -196,33 +194,48 @@
       "source": [
         "## Queue Order\n",
         "- Add priority for each data row\n",
-        "- Remove all the batch priority in your project"
+        "- Update priority for each data row"
       ],
       "cell_type": "markdown"
     },
     {
       "metadata": {},
       "source": [
-        "## See current LPOs\n",
-        "print(project.labeling_parameter_overrides().get_many(4))\n",
+        "# Select data rows from batches \n",
+        "data_rows = []\n",
+        "for batch in list(project.batches()):\n",
+        "  for data_row in batch.export_data_rows(): \n",
+        "    data_rows.append(data_row)\n",
         "\n",
+        "# Get label parameter overrides (LPOs)\n",
+        "project_lpos = list(project.labeling_parameter_overrides())\n",
         "\n",
-        "## Select data rows from batches \n",
-        "data_rows = []\n",
-        "for b in list(project.batches()):\n",
-        "  for dr in b.export_data_rows(): \n",
-        "    data_rows.append(dr)\n",
-        "  \n",
-        "## Add LPOs\n",
-        "lpos1 = []\n",
-        "p=1;\n",
-        "for dr in data_rows: \n",
-        "  lpos1.append((dr, p, 1))\n",
-        "  p+=1\n",
+        "for lpo in project_lpos:\n",
+        "  print(lpo)\n",
+        "  print(\"Data row:\", lpo.data_row().uid)"
+      ],
+      "cell_type": "code",
+      "outputs": [],
+      "execution_count": null
+    },
+    {
+      "metadata": {},
+      "source": [
+        "# Add LPOs\n",
+        "lpos = []\n",
+        "priority=1\n",
+        "for data_row in data_rows: \n",
+        "  lpos.append((data_row, priority, 1))\n",
+        "  priority+=1\n",
         "\n",
         "\n",
-        "project.set_labeling_parameter_overrides(lpos1)\n",
-        "# Get the project's LPOs"
+        "project.set_labeling_parameter_overrides(lpos)\n",
+        "\n",
+        "# Check results\n",
+        "project_lpos = list(project.labeling_parameter_overrides())\n",
+        "\n",
+        "for lpo in project_lpos:\n",
+        "  print(lpo)"
       ],
       "cell_type": "code",
       "outputs": [],
@@ -231,14 +244,18 @@
     {
       "metadata": {},
       "source": [
-        "## Verify LPOs\n",
+        "# Update LPOs\n",
+        "global_keys = []\n",
+        "for data_row in data_rows:\n",
+        "    global_keys.append(data_row.global_key)\n",
+        "\n",
+        "project.update_data_row_labeling_priority(data_rows=lb.GlobalKeys(global_keys), priority=1)\n",
+        "\n",
+        "# Check results\n",
         "project_lpos = list(project.labeling_parameter_overrides())\n",
-        "## Remove LPOs\n",
-        "# project.unset_labeling_parameter_overrides(dataset.export_data_rows())\n",
         "\n",
         "for lpo in project_lpos:\n",
-        "  print(lpo)\n",
-        "  print(\"Data row:\", lpo.data_row().uid)"
+        "  print(lpo)"
       ],
       "cell_type": "code",
       "outputs": [],

labelbox/__init__.py

@@ -1,6 +1,6 @@
 name = "labelbox"
 
-__version__ = "3.58.1"
+__version__ = "3.59.0"
 
 from labelbox.client import Client
 from labelbox.schema.project import Project
@@ -34,3 +34,4 @@
 from labelbox.schema.queue_mode import QueueMode
 from labelbox.schema.task_queue import TaskQueue
 from labelbox.schema.identifiables import UniqueIds, GlobalKeys, DataRowIds
+from labelbox.schema.identifiable import UniqueId, GlobalKey

labelbox/client.py

@@ -183,6 +183,7 @@ def convert_value(value):
 
         endpoint = self.endpoint if not experimental else self.endpoint.replace(
             "/graphql", "/_gql")
+
         try:
             request = {
                 'url': endpoint,

labelbox/data/annotation_types/base_annotation.py

@@ -1,6 +1,6 @@
 import abc
-from uuid import UUID, uuid4
-from typing import Any, Dict
+from uuid import UUID
+from typing import Any, Dict, Optional
 from pydantic import PrivateAttr
 
 from .feature import FeatureSchema
@@ -9,10 +9,10 @@
 class BaseAnnotation(FeatureSchema, abc.ABC):
     """ Base annotation class. Shouldn't be directly instantiated
     """
-    _uuid: UUID = PrivateAttr()
+    _uuid: Optional[UUID] = PrivateAttr()
     extra: Dict[str, Any] = {}
 
     def __init__(self, **data):
         super().__init__(**data)
         extra_uuid = data.get("extra", {}).get("uuid")
-        self._uuid = data.get("_uuid") or extra_uuid or uuid4()
+        self._uuid = data.get("_uuid") or extra_uuid or None

labelbox/data/serialization/ndjson/converter.py

@@ -1,7 +1,9 @@
 import logging
+import uuid
 from typing import Any, Dict, Generator, Iterable
 
 from ...annotation_types.collection import LabelCollection, LabelGenerator
+from ...annotation_types.relationship import RelationshipAnnotation
 from .label import NDLabel
 
 logger = logging.getLogger(__name__)
@@ -21,7 +23,7 @@ def deserialize(json_data: Iterable[Dict[str, Any]]) -> LabelGenerator:
         Returns:
             LabelGenerator containing the ndjson data.
         """
-        data = NDLabel(**{'annotations': json_data})
+        data = NDLabel(**{"annotations": json_data})
         res = data.to_common()
         return res
 
@@ -40,8 +42,50 @@ def serialize(
         Returns:
             A generator for accessing the ndjson representation of the data
         """
+        used_annotation_uuids = set()
+        for label in labels:
+            annotation_uuid_to_generated_uuid_lookup = {}
+            # UUIDs are private properties used to enhance UX when defining relationships.
+            # They are created for all annotations, but only utilized for relationships.
+            # To avoid overwriting, UUIDs must be unique across labels.
+            # Non-relationship annotation UUIDs are dropped (server-side generation will occur).
+            # For relationship annotations, new UUIDs are generated and stored in a lookup table.
+            for annotation in label.annotations:
+                if isinstance(annotation, RelationshipAnnotation):
+                    source_uuid = annotation.value.source._uuid
+                    target_uuid = annotation.value.target._uuid
+
+                    if (len(
+                            used_annotation_uuids.intersection(
+                                {source_uuid, target_uuid})) > 0):
+                        new_source_uuid = uuid.uuid4()
+                        new_target_uuid = uuid.uuid4()
+
+                        annotation_uuid_to_generated_uuid_lookup[
+                            source_uuid] = new_source_uuid
+                        annotation_uuid_to_generated_uuid_lookup[
+                            target_uuid] = new_target_uuid
+                        annotation.value.source._uuid = new_source_uuid
+                        annotation.value.target._uuid = new_target_uuid
+                    else:
+                        annotation_uuid_to_generated_uuid_lookup[
+                            source_uuid] = source_uuid
+                        annotation_uuid_to_generated_uuid_lookup[
+                            target_uuid] = target_uuid
+                    used_annotation_uuids.add(annotation._uuid)
+
+            for annotation in label.annotations:
+                if (not isinstance(annotation, RelationshipAnnotation) and
+                        hasattr(annotation, "_uuid")):
+                    annotation._uuid = annotation_uuid_to_generated_uuid_lookup.get(
+                        annotation._uuid, annotation._uuid)
+
         for example in NDLabel.from_common(labels):
-            res = example.dict(by_alias=True)
+            annotation_uuid = getattr(example, "uuid", None)
+
+            res = example.dict(
+                by_alias=True,
+                exclude={"uuid"} if annotation_uuid == "None" else None)
             for k, v in list(res.items()):
                 if k in IGNORE_IF_NONE and v is None:
                     del res[k]

labelbox/schema/__init__.py

@@ -22,3 +22,4 @@
 import labelbox.schema.iam_integration
 import labelbox.schema.media_type
 import labelbox.schema.identifiables
+import labelbox.schema.identifiable

labelbox/schema/annotation_import.py

@@ -23,6 +23,7 @@
     from labelbox.types import Label
 
 NDJSON_MIME_TYPE = "application/x-ndjson"
+LABEL_LIMIT = 5000
 logger = logging.getLogger(__name__)
 
 
@@ -153,6 +154,9 @@ def _get_ndjson_from_objects(cls, objects: Union[List[Dict[str, Any]],
             )
 
         objects = serialize_labels(objects)
+        if len(objects) > LABEL_LIMIT:
+            raise ValueError(
+                f"Label count {len(objects)} exceeds limit of {(LABEL_LIMIT)}")
         cls._validate_data_rows(objects)
 
         data_str = parser.dumps(objects)
@@ -502,8 +506,8 @@ def create_from_objects(
         Returns:
             MALPredictionImport
         """
-        data = cls._get_ndjson_from_objects(predictions, 'annotations')
 
+        data = cls._get_ndjson_from_objects(predictions, 'annotations')
         if len(predictions) > 0 and isinstance(predictions[0], Dict):
             predictions_dicts = cast(List[Dict[str, Any]], predictions)
             has_confidence = LabelsConfidencePresenceChecker.check(