webhook validation. Update tests

Author: Matt Sokoloff

labelbox/schema/asset_metadata.py

@@ -1,3 +1,5 @@
+from enum import Enum
+
 from labelbox.orm.db_object import DbObject
 from labelbox.orm.model import Field
 
@@ -10,12 +12,15 @@ class AssetMetadata(DbObject):
         meta_value (str): URL to an external file or a string of text
     """
 
-    VIDEO = "VIDEO"
-    IMAGE = "IMAGE"
-    TEXT = "TEXT"
-    IMAGE_OVERLAY = "IMAGE_OVERLAY"
+    class MetaType(Enum):
+        VIDEO = "VIDEO"
+        IMAGE = "IMAGE"
+        TEXT = "TEXT"
+        IMAGE_OVERLAY = "IMAGE_OVERLAY"
 
-    SUPPORTED_TYPES = {VIDEO, IMAGE, TEXT, IMAGE_OVERLAY}
+    #For backwards compatibility
+    for topic in MetaType:
+        vars()[topic.name] = topic.value
 
     meta_type = Field.String("meta_type")
     meta_value = Field.String("meta_value")

labelbox/schema/data_row.py

@@ -55,14 +55,17 @@ def create_metadata(self, meta_type, meta_value):
 
         Args:
             meta_type (str): Asset metadata type, must be one of:
-                VIDEO, IMAGE, TEXT, IMAGE_OVERLAY (AssetMetadata.SUPPORTED_TYPES)
+                VIDEO, IMAGE, TEXT, IMAGE_OVERLAY (AssetMetadata.MetaType)
             meta_value (str): Asset metadata value.
         Returns:
             `AssetMetadata` DB object.
+        Raises:
+            ValueError: meta_type must be one of the supported types.
         """
-        if meta_type not in AssetMetadata.SUPPORTED_TYPES:
+        supported_meta_types = [x.value for x in AssetMetadata.MetaType]
+        if meta_type not in supported_meta_types:
             raise ValueError(
-                f"metadata type must be one of {AssetMetadata.SUPPORTED_TYPES}. Found {meta_type}"
+                f"metadata type must be one of {supported_meta_types}. Found {meta_type}"
             )
 
         meta_type_param = "metaType"

labelbox/schema/dataset.py

@@ -170,6 +170,7 @@ def data_rows_for_external_id(self, external_id, limit=10):
 
         Args:
             external_id (str): External ID of the sought `DataRow`.
+            limit (int): The maximum number of data rows to return for the given external_id
 
         Returns:
             A single `DataRow` with the given ID.

labelbox/schema/project.py

@@ -1,15 +1,15 @@
-from collections import namedtuple
-from datetime import datetime, timezone
 import json
-from labelbox.schema.data_row import DataRow
+import time
 import logging
+from collections import namedtuple
+from datetime import datetime, timezone
 from pathlib import Path
-import time
 from typing import Dict, List, Union, Iterable
 from urllib.parse import urlparse
 
 from labelbox import utils
 from labelbox.schema.bulk_import_request import BulkImportRequest
+from labelbox.schema.data_row import DataRow
 from labelbox.exceptions import InvalidQueryError
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject, Updateable, Deletable
@@ -320,50 +320,28 @@ def validate_labeling_parameter_overrides(self, data):
         for idx, row in enumerate(data):
             if len(row) != 3:
                 raise TypeError(
-                    f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items"
+                    f"Data must be a list of tuples containing a DataRow, priority (int), num_labels (int). Found {len(row)} items. Index: {idx}"
                 )
             data_row, priority, num_labels = row
             if not isinstance(data_row, DataRow):
                 raise TypeError(
-                    f"Datarow should be be of type DataRow. Found {data_row}")
+                    f"Datarow should be be of type DataRow. Found {type(data_row)}. Index: {idx}"
+                )
 
-            for name, value in [["priority", priority],
+            for name, value in [["Priority", priority],
                                 ["Number of labels", num_labels]]:
                 if not isinstance(value, int):
                     raise TypeError(
-                        f"{name} must be an int. Found {type(value)} for data_row {data_row}"
+                        f"{name} must be an int. Found {type(value)} for data_row {data_row}. Index: {idx}"
                     )
                 if value < 1:
                     raise ValueError(
-                        f"{name} must be greater than 0 for data_row {data_row}"
+                        f"{name} must be greater than 0 for data_row {data_row}. Index: {idx}"
                     )
 
     def set_labeling_parameter_overrides(self, data):
         """ Adds labeling parameter overrides to this project.
-
-        Priority:
-            * data will be labeled in priority order 
-                - lower numbers labeled first
-                - Minimum priority is 1.
-            * Priority is not the queue position.
-                - The position is determined by the relative priority.
-                - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] 
-                    will be assigned in the following order: [data_row_2, data_row_1, data_row_3]
-            * datarows with parameter overrides will appear before datarows without overrides
-            * The priority only effects items in the queue 
-                - Assigning a priority will not automatically add the item back into the queue
-        
-        Number of labels:
-            * The number times a data row should be labeled
-            * This will create duplicates in a project
-            * The queue will never assign the same datarow to a labeler more than once
-                - if the number of labels is greater than the number of labelers working on a project then
-                    the extra items will get stuck in the queue (thsi can be fixed by removing the override at any time).
-            * This can add items to the queue (even if they have already been labeled)
-                -  New copies will only be assigned to members who have not labeled that same datarow before. 
-            * Setting this to 1 will result in the default behavior (no duplicates)
-            
-    
+                
         See information on priority here:
             https://docs.labelbox.com/en/configure-editor/queue-system#reservation-system
     
@@ -372,7 +350,29 @@ def set_labeling_parameter_overrides(self, data):
 
         Args:
             data (iterable): An iterable of tuples. Each tuple must contain
-                (Union[DataRow, datarow_id], priority, numberOfLabels) for the new override.
+                (DataRow, priority<int>, number_of_labels<int>) for the new override.
+                Priority:
+                    * Data will be labeled in priority order.
+                        - Lower number priority is labeled first.
+                        - Minimum priority is 1.
+                    * Priority is not the queue position.
+                        - The position is determined by the relative priority.
+                        - Eg. [(data_row_1, 5,1), (data_row_2, 2,1), (data_row_3, 10,1)] 
+                            will be assigned in the following order: [data_row_2, data_row_1, data_row_3]
+                    * Datarows with parameter overrides will appear before datarows without overrides.
+                    * The priority only effects items in the queue.
+                        - Assigning a priority will not automatically add the item back into the queue.  
+                Number of labels:
+                    * The number times a data row should be labeled.
+                    * This will create duplicate data rows in a project (one for each number of labels).
+                    * Data rows will be sent to the queue (even if they have already been labeled).
+                        - New copies will only be assigned to members who have not labeled that same datarow before. 
+                        - Already labeled duplicates will not be sent back to the queue.
+                    * The queue will never assign the same datarow to a single labeler more than once.
+                        - If the number of labels is greater than the number of labelers working on a project then
+                            the extra items will remain in the queue (this can be fixed by removing the override at any time).
+
+                    * Setting this to 1 will result in the default behavior (no duplicates).
         Returns:
             bool, indicates if the operation was a success.
         """

labelbox/schema/webhook.py

@@ -1,4 +1,7 @@
 import logging
+from enum import Enum
+from typing import Iterable, List
+
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject, Updateable
 from labelbox.orm.model import Entity, Field, Relationship
@@ -16,35 +19,41 @@ class Webhook(DbObject, Updateable):
         created_at (datetime)
         url (str)
         topics (str): LABEL_CREATED, LABEL_UPDATED, LABEL_DELETED
+            REVIEW_CREATED, REVIEW_UPDATED, REVIEW_DELETED
         status (str): ACTIVE, INACTIVE, REVOKED
 
     """
 
-    # Status
-    ACTIVE = "ACTIVE"
-    INACTIVE = "INACTIVE"
-    REVOKED = "REVOKED"
+    class WebhookStatus(Enum):
+        ACTIVE = "ACTIVE"
+        INACTIVE = "INACTIVE"
+        REVOKED = "REVOKED"
 
-    # Topic
-    LABEL_CREATED = "LABEL_CREATED"
-    LABEL_UPDATED = "LABEL_UPDATED"
-    LABEL_DELETED = "LABEL_DELETED"
+    class WebhookTopic(Enum):
+        LABEL_CREATED = "LABEL_CREATED"
+        LABEL_UPDATED = "LABEL_UPDATED"
+        LABEL_DELETED = "LABEL_DELETED"
+        REVIEW_CREATED = "REVIEW_CREATED"
+        REVIEW_UPDATED = "REVIEW_UPDATED"
+        REVIEW_DELETED = "REVIEW_DELETED"
 
-    REVIEW_CREATED = "REVIEW_CREATED"
-    REVIEW_UPDATED = "REVIEW_UPDATED"
-    REVIEW_DELETED = "REVIEW_DELETED"
+    #For backwards compatibility
+    for topic in WebhookStatus:
+        vars()[topic.name] = topic.value
 
-    SUPPORTED_TOPICS = {
-        LABEL_CREATED, LABEL_UPDATED, LABEL_DELETED, REVIEW_CREATED,
-        REVIEW_UPDATED, REVIEW_DELETED
-    }
+    for status in WebhookTopic:
+        vars()[status.name] = status.value
 
     updated_at = Field.DateTime("updated_at")
     created_at = Field.DateTime("created_at")
     url = Field.String("url")
     topics = Field.String("topics")
     status = Field.String("status")
 
+    created_by = Relationship.ToOne("User", False, "created_by")
+    organization = Relationship.ToOne("Organization")
+    project = Relationship.ToOne("Project")
+
     @staticmethod
     def create(client, topics, url, secret, project):
         """ Creates a Webhook.
@@ -53,7 +62,7 @@ def create(client, topics, url, secret, project):
             client (Client): The Labelbox client used to connect
                 to the server.
             topics (list of str): A list of topics this Webhook should
-                get notifications for. Must be one of Webhook.SUPPORTED_TOPICS
+                get notifications for. Must be one of Webhook.WebhookTopic
             url (str): The URL to which notifications should be sent
                 by the Labelbox server.
             secret (str): A secret key used for signing notifications.
@@ -63,10 +72,14 @@ def create(client, topics, url, secret, project):
         Returns:
             A newly created Webhook.
 
+        Raises:
+            ValueError: If the topic is not one of WebhookTopic or status is not one of WebhookStatus
+
         Information on configuring your server can be found here (this is where the url points to and the secret is set).
                         https://docs.labelbox.com/en/configure-editor/webhooks-setup#setup-steps
-        
+
         """
+        Webhook.validate_topics(topics)
 
         project_str = "" if project is None \
             else ("project:{id:\"%s\"}," % project.uid)
@@ -78,30 +91,47 @@ def create(client, topics, url, secret, project):
 
         return Webhook(client, client.execute(query_str)["createWebhook"])
 
-    created_by = Relationship.ToOne("User", False, "created_by")
-    organization = Relationship.ToOne("Organization")
-    project = Relationship.ToOne("Project")
+    @staticmethod
+    def validate_topics(topics: List["Webhook.WebhookTopic"]):
+        if not isinstance(topics, list):
+            raise TypeError(
+                f"Topics must be List[Webhook.WebhookTopic]. Found `{topics}`")
+
+        for topic in topics:
+            Webhook.validate_value(topic, Webhook.WebhookTopic)
+
+    @staticmethod
+    def validate_value(value, enum):
+        supported_values = [x.value for x in enum]
+        if value not in supported_values:
+            raise ValueError(
+                f"Value `{value}` does not exist in supported values. Expected one of {supported_values}"
+            )
 
     def delete(self):
-        self.update(status="INACTIVE")
+        self.update(status=self.WebhookStatus.INACTIVE)
 
     def update(self, topics=None, url=None, status=None):
-        """ Updates this Webhook.
+        """ Updates the Webhook.
 
         Args:
-            topics (list of str): The new topics value, optional.
-            url (str): The new URL value, optional.
-            status (str): The new status value, optional.
+            topics (List[str]): The new topics.
+            url (str): The new URL value.
+            status (str): The new status.
 
         If values are set to None then there are no updates made to that field.
 
-        The following code will delete the webhook.
-        >>> self.update(status = Webhook.INACTIVE) 
-
         """
 
         # Webhook has a custom `update` function due to custom types
         # in `status` and `topics` fields.
+
+        if topics is not None:
+            self.validate_topics(topics)
+
+        if status is not None:
+            self.validate_value(status, self.WebhookStatus)
+
         topics_str = "" if topics is None \
             else "topics: {set: [%s]}" % " ".join(topics)
         url_str = "" if url is None else "url: \"%s\"" % url

tests/integration/test_asset_metadata.py

@@ -19,8 +19,10 @@ def test_asset_metadata_crud(project, dataset, rand_gen):
     assert asset.meta_value == "Value"
     assert len(list(data_row.metadata())) == 1
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         data_row.create_metadata("NOT_SUPPORTED_TYPE", "Value")
+    assert str(exc_info.value) == \
+        f"metadata type must be one of {[x.value for x in AssetMetadata.MetaType]}. Found NOT_SUPPORTED_TYPE"
 
     # Check that filtering and sorting is prettily disabled
     with pytest.raises(InvalidQueryError) as exc_info:

tests/integration/test_labeling_parameter_overrides.py

@@ -34,24 +34,34 @@ def test_labeling_parameter_overrides(project, rand_gen):
     # currently this doesn't work so the count remains 3
     assert len(list(project.labeling_parameter_overrides())) == 1
 
-    with pytest.raises(TypeError):
-        data = [(data_rows[12], "yoo", 3)]
+    with pytest.raises(TypeError) as exc_info:
+        data = [(data_rows[12], "a_string", 3)]
         project.set_labeling_parameter_overrides(data)
+    assert str(exc_info.value) == \
+        f"Priority must be an int. Found <class 'str'> for data_row {data_rows[12]}. Index: 0"
 
-    with pytest.raises(TypeError):
-        data = [(data_rows[12], 3, "yoo")]
+    with pytest.raises(TypeError) as exc_info:
+        data = [(data_rows[12], 3, "a_string")]
         project.set_labeling_parameter_overrides(data)
+    assert str(exc_info.value) == \
+        f"Number of labels must be an int. Found <class 'str'> for data_row {data_rows[12]}. Index: 0"
 
-    with pytest.raises(TypeError):
+    with pytest.raises(TypeError) as exc_info:
         data = [(data_rows[12].uid, 1, 3)]
         project.set_labeling_parameter_overrides(data)
+    assert str(exc_info.value) == \
+        "Datarow should be be of type DataRow. Found <class 'str'>. Index: 0"
 
-    with pytest.raises(TypeError):
-        data = [(data_rows[12].uid, 0, 3)]
+    with pytest.raises(ValueError) as exc_info:
+        data = [(data_rows[12], 0, 3)]
         project.set_labeling_parameter_overrides(data)
+    assert str(exc_info.value) == \
+        f"Priority must be greater than 0 for data_row {data_rows[12]}. Index: 0"
 
-    with pytest.raises(TypeError):
-        data = [(data_rows[12].uid, 1, 0)]
+    with pytest.raises(ValueError) as exc_info:
+        data = [(data_rows[12], 1, 0)]
         project.set_labeling_parameter_overrides(data)
+    assert str(exc_info.value) == \
+        f"Number of labels must be greater than 0 for data_row {data_rows[12]}. Index: 0"
 
     dataset.delete()

tests/integration/test_project.py

@@ -60,11 +60,13 @@ def test_project_filtering(client, rand_gen):
 def test_upsert_review_queue(project):
     project.upsert_review_queue(0.6)
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         project.upsert_review_queue(1.001)
+    assert str(exc_info.value) == "Quota factor must be in the range of [0,1]"
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError) as exc_info:
         project.upsert_review_queue(-0.001)
+    assert str(exc_info.value) == "Quota factor must be in the range of [0,1]"
 
 
 def test_extend_reservations(project):
@@ -95,6 +97,6 @@ def test_attach_instructions(client, project):
         list(project.labeling_frontend_options())
         [-1].customization_options).get('projectInstructions') is not None
 
-    with pytest.raises(ValueError) as execinfo:
+    with pytest.raises(ValueError) as exc_info:
         project.upsert_instructions('/tmp/file.invalid_file_extension')
-    assert "instructions_file must end with one of" in str(execinfo.value)
+    assert "instructions_file must end with one of" in str(exc_info.value)