Merge pull request #508 from Labelbox/develop

3.17.0

Author: msokoloff1

CHANGELOG.md

@@ -1,10 +1,23 @@
 # Changelog
+
+# Version 3.17.0 (2022-03-22)
+## Added
+* Create batches from the SDK (Beta). Learn more about [batches](https://docs.labelbox.com/docs/batches)
+* Support for precision and recall metrics on Entity annotations
+
+## Fix
+* `client.create_project` type hint added for its return type
+
+## Updated
+* Removed batch MVP code
+
 # Version 3.16.0 (2022-03-08)
 ## Added
 * Ability to fetch a model run with `client.get_model_run()`
 * Ability to fetch labels from a model run with `model_run.export_labels()`
     - Note: this is only Experimental. To use, client param `enable_experimental` should
     be set to true 
+* Ability to delete an attachment
 
 ## Fix
 * Logger level is no longer set to INFO

examples/annotation_types/basics.ipynb

@@ -140,7 +140,6 @@
                 "    - Covered in this notebook\n",
                 "2. Export labels to an annotation generator\n",
                 "    - `project.label_generator()`\n",
-                "    - `project.video_label_generator()`\n",
                 "3. Use a converter to load from another format\n",
                 "    - Covered in the converters.ipynb notebook."
             ]

examples/annotation_types/converters.ipynb

@@ -35,7 +35,6 @@
                 "* The goal is to create a set of converts that convert to and from labelbox annotation types.\n",
                 "* This is automatically used when exporting labels from labelbox with:\n",
                 "    1. `label.label_generator()`\n",
-                "    2. `label.video_label_generator()`\n",
                 "* Currently we support:\n",
                 "    1. NDJson Converter\n",
                 "        - Convert to and from the prediction import format (mea, mal)\n",
@@ -363,7 +362,7 @@
             ],
             "source": [
                 "project = client.get_project(\"ckqcx1d58068c0y619qv7hzgu\")\n",
-                "labels = project.video_label_generator()\n",
+                "labels = project.label_generator()\n",
                 "\n",
                 "for label in labels:\n",
                 "    annotation_lookup = label.frame_annotations()\n",

labelbox/__init__.py

@@ -1,14 +1,15 @@
 name = "labelbox"
-__version__ = "3.16.0"
+__version__ = "3.17.0"
 
-from labelbox.schema.project import Project
 from labelbox.client import Client
+from labelbox.schema.project import Project
 from labelbox.schema.model import Model
 from labelbox.schema.bulk_import_request import BulkImportRequest
 from labelbox.schema.annotation_import import MALPredictionImport, MEAPredictionImport, LabelImport
 from labelbox.schema.dataset import Dataset
 from labelbox.schema.data_row import DataRow
 from labelbox.schema.label import Label
+from labelbox.schema.batch import Batch
 from labelbox.schema.review import Review
 from labelbox.schema.user import User
 from labelbox.schema.organization import Organization

labelbox/client.py

@@ -404,7 +404,7 @@ def _get_single(self, db_object_type, uid):
         else:
             return db_object_type(self, res)
 
-    def get_project(self, project_id):
+    def get_project(self, project_id) -> Project:
         """ Gets a single Project with the given ID.
 
             >>> project = client.get_project("<project_id>")
@@ -918,4 +918,4 @@ def get_model_run(self, model_run_id: str) -> ModelRun:
         Returns:
             A ModelRun object.
         """
-        return self._get_single(Entity.ModelRun, model_run_id)
+        return self._get_single(Entity.ModelRun, model_run_id)

labelbox/data/metrics/confusion_matrix/calculation.py

@@ -2,9 +2,9 @@
 
 import numpy as np
 
-from ..iou.calculation import _get_mask_pairs, _get_vector_pairs, miou
+from ..iou.calculation import _get_mask_pairs, _get_vector_pairs, _get_ner_pairs, miou
 from ...annotation_types import (ObjectAnnotation, ClassificationAnnotation,
-                                 Mask, Geometry, Checklist, Radio,
+                                 Mask, Geometry, Checklist, Radio, TextEntity,
                                  ScalarMetricValue, ConfusionMatrixMetricValue)
 from ..group import (get_feature_pairs, get_identifying_key, has_no_annotations,
                      has_no_matching_annotations)
@@ -68,6 +68,9 @@ def feature_confusion_matrix(
     elif isinstance(predictions[0].value, Geometry):
         return vector_confusion_matrix(ground_truths, predictions,
                                        include_subclasses, iou)
+    elif isinstance(predictions[0].value, TextEntity):
+        return ner_confusion_matrix(ground_truths, predictions,
+                                    include_subclasses, iou)
     elif isinstance(predictions[0], ClassificationAnnotation):
         return classification_confusion_matrix(ground_truths, predictions)
     else:
@@ -288,3 +291,23 @@ def mask_confusion_matrix(ground_truths: List[ObjectAnnotation],
     fn_mask = (prediction_np == 0) & (ground_truth_np == 1)
     tn_mask = prediction_np == ground_truth_np == 0
     return [np.sum(tp_mask), np.sum(fp_mask), np.sum(fn_mask), np.sum(tn_mask)]
+
+
+def ner_confusion_matrix(ground_truths: List[ObjectAnnotation],
+                         predictions: List[ObjectAnnotation],
+                         include_subclasses: bool,
+                         iou: float) -> Optional[ConfusionMatrixMetricValue]:
+    """Computes confusion matrix metric between two lists of TextEntity objects
+
+    Args:
+        ground_truths: List of ground truth mask annotations
+        predictions: List of prediction mask annotations
+    Returns:
+        confusion matrix as a list: [TP,FP,TN,FN]
+    """
+    if has_no_matching_annotations(ground_truths, predictions):
+        return [0, int(len(predictions) > 0), 0, int(len(ground_truths) > 0)]
+    elif has_no_annotations(ground_truths, predictions):
+        return None
+    pairs = _get_ner_pairs(ground_truths, predictions)
+    return object_pair_confusion_matrix(pairs, include_subclasses, iou)

labelbox/data/metrics/iou/calculation.py

@@ -7,7 +7,7 @@
 from ..group import get_feature_pairs, get_identifying_key, has_no_annotations, has_no_matching_annotations
 from ...annotation_types import (ObjectAnnotation, ClassificationAnnotation,
                                  Mask, Geometry, Point, Line, Checklist, Text,
-                                 Radio, ScalarMetricValue)
+                                 TextEntity, Radio, ScalarMetricValue)
 
 
 def miou(ground_truths: List[Union[ObjectAnnotation, ClassificationAnnotation]],
@@ -61,6 +61,8 @@ def feature_miou(ground_truths: List[Union[ObjectAnnotation,
         return vector_miou(ground_truths, predictions, include_subclasses)
     elif isinstance(predictions[0], ClassificationAnnotation):
         return classification_miou(ground_truths, predictions)
+    elif isinstance(predictions[0].value, TextEntity):
+        return ner_miou(ground_truths, predictions, include_subclasses)
     else:
         raise ValueError(
             f"Unexpected annotation found. Found {type(predictions[0].value)}")
@@ -269,3 +271,51 @@ def _ensure_valid_poly(poly):
 def _mask_iou(mask1: np.ndarray, mask2: np.ndarray) -> ScalarMetricValue:
     """Computes iou between two binary segmentation masks."""
     return np.sum(mask1 & mask2) / np.sum(mask1 | mask2)
+
+
+def _get_ner_pairs(
+    ground_truths: List[ObjectAnnotation], predictions: List[ObjectAnnotation]
+) -> List[Tuple[ObjectAnnotation, ObjectAnnotation, ScalarMetricValue]]:
+    """Get iou score for all possible pairs of ground truths and predictions"""
+    pairs = []
+    for ground_truth, prediction in product(ground_truths, predictions):
+        score = _ner_iou(ground_truth.value, prediction.value)
+        pairs.append((ground_truth, prediction, score))
+    return pairs
+
+
+def _ner_iou(ner1: TextEntity, ner2: TextEntity):
+    """Computes iou between two text entity annotations"""
+    intersection_start, intersection_end = max(ner1.start, ner2.start), min(
+        ner1.end, ner2.end)
+    union_start, union_end = min(ner1.start,
+                                 ner2.start), max(ner1.end, ner2.end)
+    #edge case of only one character in text
+    if union_start == union_end:
+        return 1
+    #if there is no intersection
+    if intersection_start > intersection_end:
+        return 0
+    return (intersection_end - intersection_start) / (union_end - union_start)
+
+
+def ner_miou(ground_truths: List[ObjectAnnotation],
+             predictions: List[ObjectAnnotation],
+             include_subclasses: bool) -> Optional[ScalarMetricValue]:
+    """
+    Computes iou score for all features with the same feature schema id.
+    Calculation includes subclassifications.
+
+    Args:
+        ground_truths: List of ground truth ner annotations
+        predictions: List of prediction ner annotations
+    Returns:
+        float representing the iou score for the feature type.
+         If there are no matches then this returns none
+    """
+    if has_no_matching_annotations(ground_truths, predictions):
+        return 0.
+    elif has_no_annotations(ground_truths, predictions):
+        return None
+    pairs = _get_ner_pairs(ground_truths, predictions)
+    return object_pair_miou(pairs, include_subclasses)

labelbox/orm/model.py

@@ -347,6 +347,7 @@ class Entity(metaclass=EntityMeta):
         Invite: Type[labelbox.Invite]
         InviteLimit: Type[labelbox.InviteLimit]
         ProjectRole: Type[labelbox.ProjectRole]
+        Batch: Type[labelbox.Batch]
 
     @classmethod
     def _attributes_of_type(cls, attr_type):

labelbox/schema/__init__.py

@@ -18,4 +18,5 @@
 import labelbox.schema.user
 import labelbox.schema.webhook
 import labelbox.schema.data_row_metadata
+import labelbox.schema.batch
 import labelbox.schema.iam_integration

labelbox/schema/batch.py

@@ -0,0 +1,25 @@
+from labelbox.orm.db_object import DbObject
+from labelbox.orm.model import Field, Relationship
+
+
+class Batch(DbObject):
+    """ A Batch is a group of data rows submitted to a project for labeling
+
+    Attributes:
+        name (str)
+        created_at (datetime)
+        updated_at (datetime)
+        deleted (bool)
+
+        project (Relationship): `ToOne` relationship to Project
+        created_by (Relationship): `ToOne` relationship to User
+
+    """
+    name = Field.String("name")
+    created_at = Field.DateTime("created_at")
+    updated_at = Field.DateTime("updated_at")
+    size = Field.Int("size")
+
+    # Relationships
+    project = Relationship.ToOne("Project")
+    created_by = Relationship.ToOne("User")