docs and validation

Author: Matt Sokoloff

labelbox/schema/asset_metadata.py

@@ -10,12 +10,12 @@ class AssetMetadata(DbObject):
         meta_value (str): URL to an external file or a string of text
     """
 
-    VALID_TYPES = {
-        "VIDEO",
-        "IMAGE",
-        "TEXT",
-        "IMAGE_OVERLAY"
-    }
+    VIDEO = "VIDEO"
+    IMAGE = "IMAGE"
+    TEXT= "TEXT"
+    IMAGE_OVERLAY = "IMAGE_OVERLAY"
+    
+    SUPPORTED_TYPES = {VIDEO, IMAGE, TEXT, IMAGE_OVERLAY}
 
     meta_type = Field.String("meta_type")
     meta_value = Field.String("meta_value")

labelbox/schema/bulk_import_request.py

@@ -493,7 +493,7 @@ class Config:
         extra = 'forbid'
 
         @staticmethod
-        def determinants(parent_cls) -> None:
+        def determinants(parent_cls) -> List[str]:
             #This is a hack for better error messages
             return [
                 k for k, v in parent_cls.__fields__.items()

labelbox/schema/data_row.py

@@ -1,7 +1,6 @@
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject, Updateable, BulkDeletable
 from labelbox.orm.model import Entity, Field, Relationship
-from labelbox.pagination import PaginatedCollection
 from labelbox.schema.asset_metadata import AssetMetadata
 
 
@@ -34,7 +33,6 @@ class DataRow(DbObject, Updateable, BulkDeletable):
     labels = Relationship.ToMany("Label", True)
     metadata = Relationship.ToMany("AssetMetadata", False, "metadata")
     predictions = Relationship.ToMany("Prediction", False)
-
 
 
     @staticmethod
@@ -64,7 +62,7 @@ def create_metadata(self, meta_type, meta_value):
             `AssetMetadata` DB object.
         """
         if meta_type not in AssetMetadata.VALID_TYPES:
-            raise ValueError(f"metadata must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}")
+            raise ValueError(f"metadata type must be one of {AssetMetadata.VALID_TYPES}. Found {meta_type}")
 
         meta_type_param = "metaType"
         meta_value_param = "metaValue"

labelbox/schema/dataset.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from multiprocessing.dummy import Pool as ThreadPool
 import os
 
@@ -163,7 +164,7 @@ def convert_item(item):
         task._user = user
         return task
 
-    def data_row_for_external_id(self, external_id):
+    def data_rows_for_external_id(self, external_id, limit = 10):
         """ Convenience method for getting a single `DataRow` belonging to this
         `Dataset` that has the given `external_id`.
 
@@ -182,10 +183,29 @@ def data_row_for_external_id(self, external_id):
         where = DataRow.external_id == external_id
 
         data_rows = self.data_rows(where=where)
-        # Get at most two data_rows.
-        data_rows = [row for row, _ in zip(data_rows, range(2))]
+        # Get at most `limit` data_rows.
+        data_rows = [row for row, _ in zip(data_rows, range(limit))]
 
-        if len(data_rows) != 1:
+        if not len(data_rows):
             raise ResourceNotFoundError(DataRow, where)
+        return data_rows
+
+    def data_row_for_external_id(self, external_id):
+        """ Convenience method for getting a single `DataRow` belonging to this
+        `Dataset` that has the given `external_id`.
+
+        Args:
+            external_id (str): External ID of the sought `DataRow`.
 
+        Returns:
+            A single `DataRow` with the given ID.
+
+        Raises:
+            labelbox.exceptions.ResourceNotFoundError: If there is no `DataRow`
+                in this `DataSet` with the given external ID, or if there are
+                multiple `DataRows` for it.
+        """
+        data_rows = self.data_rows_for_external_id(external_id=external_id, limit = 2)
+        if len(data_rows) > 1:
+                logging.warn("More than one data_row has the provided external_id. Use function data_rows_for_external_id to fetch all")
         return data_rows[0]

labelbox/schema/project.py

@@ -1,6 +1,7 @@
 from collections import namedtuple
 from datetime import datetime, timezone
 import json
+from labelbox.schema.data_row import DataRow
 import logging
 from pathlib import Path
 import time
@@ -88,6 +89,7 @@ def create_label(self, **kwargs):
         # deprecated and we don't want the Py client lib user to know
         # about them. At the same time they're connected to a Label at
         # label creation in a non-standard way (connect via name).
+        logger.warning("This function is deprecated and is not compatible with the new editor.")
 
         Label = Entity.Label
 
@@ -249,18 +251,56 @@ def setup(self, labeling_frontend, labeling_frontend_options):
         timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
         self.update(setup_complete=timestamp)
 
+    def validate_labeling_parameter_overrides(self, data):
+        for idx, row in enumerate(data):
+            if len(row) != 3:
+                raise TypeError(f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items")
+            data_row, priority, num_labels = row
+            if not isinstance(data_row, DataRow):
+                raise TypeError(f"Datarow should be be of type DataRow. Found {data_row}")
+
+            for name, value in [["priority", priority], ["Number of labels", num_labels]]:
+                if not isinstance(value, int):
+                    raise TypeError(f"{name} must be an int. Found {type(value)} for data_row {data_row}")
+                if value < 1:
+                    raise ValueError(f"{name} must be greater than 0 for data_row {data_row}")
+
     def set_labeling_parameter_overrides(self, data):
         """ Adds labeling parameter overrides to this project.
 
+        Priority:
+            * data will be labeled in priority order with the lower priority numbers being labeled first
+                - Minimum priority is 1.
+            * Priority is not the queue position. The position is determined by the relative priority.
+                - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] 
+                    will be assigned in the following order: [data_row_2, data_row_1, data_row_3]
+            * datarows with parameter overrides will appear before datarows without overrides
+            * The priority only effects items in the queue and assigning a priority will not automatically add the item back the queue 
+                - If a datarow has already been labeled this will not have an effect until it is added back into the queue
+        
+        Number of labels:
+            * The number times a data row should be labeled
+            * This will create duplicates in a project
+            * The queue will never assign the same datarow to a labeler more than once
+                - if the number of labels is greater than the number of labelers working on a project then
+                    the extra items will get stuck in the queue (thsi can be fixed by removing the override at any time).
+            * This can add items to the queue even if they have already been labeled but they will only be assigned to members who have not labeled that image before. 
+            * Set this to 1 if you only want to effect the priority.
+            
+    
+        See information on priority here:
+            https://docs.labelbox.com/en/configure-editor/queue-system#reservation-system
+    
             >>> project.set_labeling_parameter_overrides([
             >>>     (data_row_1, 2, 3), (data_row_2, 1, 4)])
 
         Args:
             data (iterable): An iterable of tuples. Each tuple must contain
-                (DataRow, priority, numberOfLabels) for the new override.
+                (Union[DataRow, datarow_id], priority, numberOfLabels) for the new override.
         Returns:
             bool, indicates if the operation was a success.
         """
+        self.validate_labeling_parameter_overrides(data)
         data_str = ",\n".join(
             "{dataRow: {id: \"%s\"}, priority: %d, numLabels: %d }" %
             (data_row.uid, priority, num_labels)
@@ -275,6 +315,8 @@ def set_labeling_parameter_overrides(self, data):
     def unset_labeling_parameter_overrides(self, data_rows):
         """ Removes labeling parameter overrides to this project.
 
+        * This will remove unlabeled duplicates in the queue.
+
         Args:
             data_rows (iterable): An iterable of DataRows.
         Returns:
@@ -290,12 +332,19 @@ def unset_labeling_parameter_overrides(self, data_rows):
         return res["project"]["unsetLabelingParameterOverrides"]["success"]
 
     def upsert_review_queue(self, quota_factor):
-        """ Reinitiates the review queue for this project.
+        """ Sets the the proportion of total assets in a project to review.
+
+        More information can be found here: 
+            https://docs.labelbox.com/en/quality-assurance/review-labels#configure-review-percentage
 
         Args:
             quota_factor (float): Which part (percentage) of the queue
                 to reinitiate. Between 0 and 1.
         """
+
+        if (quota_factor > 1.) or (quota_factor < 0.):
+            raise ValueError("Quota factor must be in the range of [0,1]")
+
         id_param = "projectId"
         quota_param = "quotaFactor"
         query_str = """mutation UpsertReviewQueuePyApi($%s: ID!, $%s: Float!){
@@ -307,25 +356,6 @@ def upsert_review_queue(self, quota_factor):
             quota_param: quota_factor
         })
 
-    def extend_reservations(self, queue_type):
-        """ Extends all the current reservations for the current user on the given
-        queue type.
-
-        Args:
-            queue_type (str): Either "LabelingQueue" or "ReviewQueue"
-        Returns:
-            int, the number of reservations that were extended.
-        """
-        if queue_type not in ("LabelingQueue", "ReviewQueue"):
-            raise InvalidQueryError("Unsupported queue type: %s" % queue_type)
-
-        id_param = "projectId"
-        query_str = """mutation ExtendReservationsPyApi($%s: ID!){
-            extendReservations(projectId:$%s queueType:%s)}""" % (
-            id_param, id_param, queue_type)
-        res = self.client.execute(query_str, {id_param: self.uid})
-        return res["extendReservations"]
-
     def create_prediction_model(self, name, version):
         """ Creates a PredictionModel connected to a Legacy Editor Project.
 
@@ -335,6 +365,9 @@ def create_prediction_model(self, name, version):
         Returns:
             A newly created PredictionModel.
         """
+
+        logger.warning("This function is deprecated and is not compatible with the new editor.")
+
         PM = Entity.PredictionModel
         model = self.client._create(PM, {
             PM.name.name: name,
@@ -360,6 +393,8 @@ def create_prediction(self, label, data_row, prediction_model=None):
                 is None and this Project's active_prediction_model is also
                 None.
         """
+        logger.warning("This function is deprecated and is not compatible with the new editor.")
+
         if prediction_model is None:
             prediction_model = self.active_prediction_model()
             if prediction_model is None:
@@ -433,6 +468,7 @@ def upload_annotations(
         Returns:
             BulkImportRequest
         """
+    
         if isinstance(annotations, str) or isinstance(annotations, Path):
 
             def _is_url_valid(url: Union[str, Path]) -> bool:
@@ -493,6 +529,8 @@ class LabelingParameterOverride(DbObject):
     number_of_labels = Field.Int("number_of_labels")
 
 
+
+
 LabelerPerformance = namedtuple(
     "LabelerPerformance", "user count seconds_per_label, total_time_labeling "
     "consensus average_benchmark_agreement last_activity_time")

labelbox/schema/webhook.py

@@ -1,7 +1,9 @@
+import logging
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject, Updateable
 from labelbox.orm.model import Entity, Field, Relationship
 
+logger = logging.getLogger(__name__)
 
 class Webhook(DbObject, Updateable):
     """ Represents a server-side rule for sending notifications to a web-server
@@ -27,6 +29,19 @@ class Webhook(DbObject, Updateable):
     LABEL_UPDATED = "LABEL_UPDATED"
     LABEL_DELETED = "LABEL_DELETED"
 
+    REVIEW_CREATED ="REVIEW_CREATED" 
+    REVIEW_UPDATED ="REVIEW_UPDATED" 
+    REVIEW_DELETED = "REVIEW_DELETED"
+
+    SUPPORTED_TOPICS = {
+        LABEL_CREATED, 
+        LABEL_UPDATED, 
+        LABEL_DELETED,
+        REVIEW_CREATED,
+        REVIEW_UPDATED,
+        REVIEW_DELETED
+    }
+
     updated_at = Field.DateTime("updated_at")
     created_at = Field.DateTime("created_at")
     url = Field.String("url")
@@ -41,7 +56,7 @@ def create(client, topics, url, secret, project):
             client (Client): The Labelbox client used to connect
                 to the server.
             topics (list of str): A list of topics this Webhook should
-                get notifications for.
+                get notifications for. Must be one of Webhook.SUPPORTED_TOPICS
             url (str): The URL to which notifications should be sent
                 by the Labelbox server.
             secret (str): A secret key used for signing notifications.
@@ -50,7 +65,12 @@ def create(client, topics, url, secret, project):
                 events in your organization.
         Returns:
             A newly created Webhook.
+
+        Information on configuring your server can be found here (this is where the url points to and the secret is set).
+                        https://docs.labelbox.com/en/configure-editor/webhooks-setup#setup-steps
+        
         """
+
         project_str = "" if project is None \
             else ("project:{id:\"%s\"}," % project.uid)
 
@@ -65,14 +85,24 @@ def create(client, topics, url, secret, project):
     organization = Relationship.ToOne("Organization")
     project = Relationship.ToOne("Project")
 
+    def delete(self):
+        self.update(status = "INACTIVE")
+
     def update(self, topics=None, url=None, status=None):
         """ Updates this Webhook.
 
         Args:
             topics (list of str): The new topics value, optional.
             url (str): The new URL value, optional.
             status (str): The new status value, optional.
+
+        If values are set to None then there are no updates made to that field.
+
+        The following code will delete the webhook.
+        >>> self.update(status = Webhook.INACTIVE) 
+
         """
+
         # Webhook has a custom `update` function due to custom types
         # in `status` and `topics` fields.
         topics_str = "" if topics is None \