Handle new editor task types without breaking Project

Author: Val Brodsky

libs/labelbox/src/labelbox/client.py

@@ -743,6 +743,10 @@ def create_project(self, **kwargs) -> Project:
         Raises:
             InvalidAttributeError: If the Project type does not contain
                 any of the attribute names given in kwargs.
+
+        NOTE: the following attributes are used only chat model evaluation projects:
+            dataset_name_or_id, append_to_existing_dataset, data_row_count, editor_task_type
+            They are not used for general projects and not supported in this method
         """
         kwargs.pop("dataset_name_or_id", None)
         kwargs.pop("append_to_existing_dataset", None)

libs/labelbox/src/labelbox/schema/ontology_kind.py

@@ -33,6 +33,21 @@ class EditorTaskType(Enum):
     def is_supported(cls, value):
         return isinstance(value, cls)
 
+    @classmethod
+    def _missing_(cls, name) -> 'EditorTaskType':
+        """Handle missing null new task types
+            Handle upper case names for compatibility with
+            the GraphQL"""
+
+        if name is None:
+            return cls.Missing
+
+        for name, member in cls.__members__.items():
+            if name == name.upper():
+                return member
+
+        return cls.Missing
+
 
 class EditorTaskTypeMapper:
 

libs/labelbox/tests/integration/conftest.py

@@ -400,7 +400,10 @@ def chat_evaluation_ontology(client, rand_gen):
 
     yield ontology
 
-    client.delete_unused_ontology(ontology.uid)
+    try:
+        client.delete_unused_ontology(ontology.uid)
+    except Exception as e:
+        print(f"Failed to delete ontology {ontology.uid}: {str(e)}")
 
 
 @pytest.fixture

libs/labelbox/tests/integration/test_offline_chat_evaluation_project.py

@@ -1,6 +1,11 @@
-def test_create_offline_chat_evaluation_project(
-        client, rand_gen, offline_chat_evaluation_project,
-        chat_evaluation_ontology, offline_conversational_data_row):
+import pytest
+
+
+def test_create_offline_chat_evaluation_project(client, rand_gen,
+                                                offline_chat_evaluation_project,
+                                                chat_evaluation_ontology,
+                                                offline_conversational_data_row,
+                                                model_config):
     project = offline_chat_evaluation_project
     assert project
 
@@ -15,3 +20,7 @@ def test_create_offline_chat_evaluation_project(
         [offline_conversational_data_row.uid],  # sample of data row objects
     )
     assert batch
+
+    # Can not add a model config to an offline chat evaluation project, since we do not use live models
+    with pytest.raises(Exception):
+        project.add_model_config(model_config.uid)

libs/labelbox/tests/unit/test_project.py

@@ -0,0 +1,36 @@
+import pytest
+from unittest.mock import MagicMock
+
+from labelbox.schema.project import Project
+from labelbox.schema.ontology_kind import EditorTaskType
+
+
+@pytest.mark.parametrize(
+    'api_editor_task_type, expected_editor_task_type',
+    [(None, EditorTaskType.Missing),
+     ('MODEL_CHAT_EVALUATION', EditorTaskType.ModelChatEvaluation),
+     ('RESPONSE_CREATION', EditorTaskType.ResponseCreation),
+     ('OFFLINE_MODEL_CHAT_EVALUATION',
+      EditorTaskType.OfflineModelChatEvaluation),
+     ('NEW_TYPE', EditorTaskType.Missing)])
+def test_project_editor_task_type(api_editor_task_type,
+                                  expected_editor_task_type):
+    client = MagicMock()
+    project = Project(
+        client, {
+            "id": "test",
+            "name": "test",
+            "createdAt": "2021-06-01T00:00:00.000Z",
+            "updatedAt": "2021-06-01T00:00:00.000Z",
+            "autoAuditNumberOfLabels": 1,
+            "autoAuditPercentage": 100,
+            "dataRowCount": 1,
+            "description": "test",
+            "editorTaskType": api_editor_task_type,
+            "lastActivityTime": "2021-06-01T00:00:00.000Z",
+            "allowedMediaType": "IMAGE",
+            "queueMode": "BATCH",
+            "setupComplete": "2021-06-01T00:00:00.000Z",
+        })
+
+    assert project.editor_task_type == expected_editor_task_type