[QQC-2474] Expose quality mode through SDK create_project method (#1224)

Author: richardsun0713

labelbox/client.py

@@ -31,6 +31,8 @@
 from labelbox.schema.model_run import ModelRun
 from labelbox.schema.ontology import Ontology, Tool, Classification, FeatureSchema
 from labelbox.schema.organization import Organization
+from labelbox.schema.quality_mode import QualityMode, BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS, \
+    BENCHMARK_AUTO_AUDIT_PERCENTAGE, CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS, CONSENSUS_AUTO_AUDIT_PERCENTAGE
 from labelbox.schema.user import User
 from labelbox.schema.project import Project
 from labelbox.schema.role import Role
@@ -636,16 +638,33 @@ def create_project(self, **kwargs) -> Project:
             description (str): A short summary for the project
             media_type (MediaType): The type of assets that this project will accept
             queue_mode (Optional[QueueMode]): The queue mode to use
-            auto_audit_percentage (Optional[float]): The percentage of data rows that will require more than 1 label
-            auto_audit_number_of_labels (Optional[float]): Number of labels required for data rows selected for multiple labeling (auto_audit_percentage)
+            quality_mode (Optional[QualityMode]): The quality mode to use (e.g. Benchmark, Consensus). Defaults to
+                Benchmark
         Returns:
             A new Project object.
         Raises:
             InvalidAttributeError: If the Project type does not contain
                 any of the attribute names given in kwargs.
         """
-        media_type = kwargs.get("media_type")
+
+        auto_audit_percentage = kwargs.get("auto_audit_percentage")
+        auto_audit_number_of_labels = kwargs.get("auto_audit_number_of_labels")
+        if auto_audit_percentage is not None or auto_audit_number_of_labels is not None:
+            raise ValueError(
+                "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
+            )
+
         queue_mode = kwargs.get("queue_mode")
+        if queue_mode is QueueMode.Dataset:
+            raise ValueError(
+                "Dataset queue mode is deprecated. Please prefer Batch queue mode."
+            )
+        elif queue_mode is QueueMode.Batch:
+            logger.warning(
+                "Passing a queue mode of batch is redundant and will soon no longer be supported."
+            )
+
+        media_type = kwargs.get("media_type")
         if media_type:
             if MediaType.is_supported(media_type):
                 media_type = media_type.value
@@ -658,20 +677,25 @@ def create_project(self, **kwargs) -> Project:
                 "Creating a project without specifying media_type"
                 " through this method will soon no longer be supported.")
 
-        if not queue_mode:
-            logger.warning(
-                "Default createProject behavior will soon be adjusted to prefer "
-                "batch projects. Pass in `queue_mode` parameter explicitly to opt-out for the "
-                "time being.")
-        elif queue_mode == QueueMode.Dataset:
-            logger.warning(
-                "QueueMode.Dataset will eventually be deprecated, and is no longer "
-                "recommended for new projects. Prefer QueueMode.Batch instead.")
+        quality_mode = kwargs.get("quality_mode")
+        if not quality_mode:
+            logger.info("Defaulting quality mode to Benchmark.")
+
+        data = kwargs
+        data.pop("quality_mode", None)
+        if quality_mode is None or quality_mode is QualityMode.Benchmark:
+            data[
+                "auto_audit_number_of_labels"] = BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS
+            data["auto_audit_percentage"] = BENCHMARK_AUTO_AUDIT_PERCENTAGE
+        else:
+            data[
+                "auto_audit_number_of_labels"] = CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS
+            data["auto_audit_percentage"] = CONSENSUS_AUTO_AUDIT_PERCENTAGE
 
         return self._create(Entity.Project, {
-            **kwargs,
+            **data,
             **({
-                'media_type': media_type
+                "media_type": media_type
             } if media_type else {})
         })
 

labelbox/orm/query.py

@@ -314,7 +314,7 @@ def relationship(source, relationship, where, order_by):
 
 
 def create(entity, data):
-    """ Generats a query and parameters for creating a new DB object.
+    """ Generates a query and parameters for creating a new DB object.
 
     Args:
         entity (type): An Entity subtype indicating which kind of

labelbox/schema/project.py

@@ -120,12 +120,10 @@ def update(self, **kwargs):
         for a project is inferred through the following attributes:
 
         Benchmark:
-            auto_audit_number_of_labels = 1
-            auto_audit_percentage = 1.0
+            auto_audit_number_of_labels = 1 and auto_audit_percentage = 1.0
 
         Consensus:
-            auto_audit_number_of_labels > 1
-            auto_audit_percentage <= 1.0
+            auto_audit_number_of_labels > 1 or auto_audit_percentage <= 1.0
 
         Attempting to switch between benchmark and consensus modes is an invalid operation and will result
         in an error.

labelbox/schema/quality_mode.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class QualityMode(str, Enum):
+    Benchmark = "BENCHMARK"
+    Consensus = "CONSENSUS"
+
+
+BENCHMARK_AUTO_AUDIT_NUMBER_OF_LABELS = 1
+BENCHMARK_AUTO_AUDIT_PERCENTAGE = 1
+CONSENSUS_AUTO_AUDIT_NUMBER_OF_LABELS = 3
+CONSENSUS_AUTO_AUDIT_PERCENTAGE = 0

tests/integration/conftest.py

@@ -22,6 +22,7 @@
 from labelbox.schema.enums import AnnotationImportState
 from labelbox.schema.invite import Invite
 from labelbox.schema.project import Project
+from labelbox.schema.quality_mode import QualityMode
 from labelbox.schema.queue_mode import QueueMode
 from labelbox.schema.user import User
 
@@ -231,7 +232,7 @@ def project(client, rand_gen):
 @pytest.fixture
 def consensus_project(client, rand_gen):
     project = client.create_project(name=rand_gen(str),
-                                    auto_audit_percentage=0,
+                                    quality_mode=QualityMode.Consensus,
                                     queue_mode=QueueMode.Batch,
                                     media_type=MediaType.Image)
     yield project

tests/integration/test_legacy_project.py

@@ -1,15 +1,13 @@
 import pytest
 
-from labelbox.exceptions import InvalidQueryError, MalformedQueryException
-from labelbox.schema.media_type import MediaType
 from labelbox.schema.queue_mode import QueueMode
 
 
 def test_project_dataset(client, rand_gen):
     with pytest.raises(
-            MalformedQueryException,
+            ValueError,
             match=
-            "DataSet queue mode is deprecated. Please prefer Batch queue mode."
+            "Dataset queue mode is deprecated. Please prefer Batch queue mode."
     ):
         client.create_project(
             name=rand_gen(str),
@@ -18,6 +16,21 @@ def test_project_dataset(client, rand_gen):
 
 
 def test_legacy_project_dataset_relationships(project, dataset):
-
     assert [ds for ds in project.datasets()] == []
     assert [p for p in dataset.projects()] == []
+
+
+def test_project_auto_audit_parameters(client, rand_gen):
+    with pytest.raises(
+            ValueError,
+            match=
+            "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
+    ):
+        client.create_project(name=rand_gen(str), auto_audit_percentage=0.5)
+
+    with pytest.raises(
+            ValueError,
+            match=
+            "quality_mode must be set instead of auto_audit_percentage or auto_audit_number_of_labels."
+    ):
+        client.create_project(name=rand_gen(str), auto_audit_number_of_labels=2)

tests/integration/test_project.py

@@ -8,11 +8,11 @@
 from labelbox import Project, LabelingFrontend, Dataset
 from labelbox.exceptions import InvalidQueryError
 from labelbox.schema.media_type import MediaType
+from labelbox.schema.quality_mode import QualityMode
 from labelbox.schema.queue_mode import QueueMode
 
 
 def test_project(client, rand_gen):
-
     data = {
         "name": rand_gen(str),
         "description": rand_gen(str),
@@ -260,3 +260,19 @@ def test_media_type(client, project: Project, rand_gen):
                                         media_type=MediaType[media_type])
         assert project.media_type == MediaType[media_type]
         project.delete()
+
+
+def test_queue_mode(client, rand_gen):
+    project = client.create_project(name=rand_gen(str))  # defaults to benchmark
+    assert project.auto_audit_number_of_labels == 1
+    assert project.auto_audit_percentage == 1
+
+    project = client.create_project(name=rand_gen(str),
+                                    quality_mode=QualityMode.Benchmark)
+    assert project.auto_audit_number_of_labels == 1
+    assert project.auto_audit_percentage == 1
+
+    project = client.create_project(name=rand_gen(str),
+                                    quality_mode=QualityMode.Consensus)
+    assert project.auto_audit_number_of_labels == 3
+    assert project.auto_audit_percentage == 0