AL-3578: Added MAL and LI warning message

Author: dubininsergey

labelbox/data/annotation_types/label.py

@@ -62,8 +62,8 @@ def frame_annotations(
         frame_dict = defaultdict(list)
         for annotation in self.annotations:
             if isinstance(
-                    annotation,
-                (VideoObjectAnnotation, VideoClassificationAnnotation)):
+                annotation,
+                    (VideoObjectAnnotation, VideoClassificationAnnotation)):
                 frame_dict[annotation.frame].append(annotation)
         return frame_dict
 
@@ -151,7 +151,8 @@ def assign_feature_schema_ids(
             elif isinstance(annotation, ObjectAnnotation):
                 self._assign_or_raise(annotation, tool_lookup)
                 for classification in annotation.classifications:
-                    self._assign_or_raise(classification, classification_lookup)
+                    self._assign_or_raise(
+                        classification, classification_lookup)
                     self._assign_option(classification, classification_lookup)
             else:
                 raise TypeError(

labelbox/data/mixins.py

@@ -10,6 +10,8 @@ class ConfidenceMixin(BaseModel):
 
     @validator('confidence')
     def confidence_valid_float(cls, value):
+        if value is None:
+            return value
         if not isinstance(value, (int, float)) or not 0 <= value <= 1:
             raise ValueError('must be float within [0,1] range')
         return value

labelbox/schema/annotation_import.py

@@ -3,17 +3,19 @@
 import logging
 import os
 import time
-from typing import Any, Dict, List, BinaryIO
-from tqdm import tqdm  # type: ignore
+from typing import Any, BinaryIO, Dict, List
 
 import backoff
 import ndjson
 import requests
+from tqdm import tqdm  # type: ignore
 
 import labelbox
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject
 from labelbox.orm.model import Field, Relationship
+from labelbox.schema.annotation_import_validators import \
+    LabelsConfidencePresenceChecker
 from labelbox.schema.enums import AnnotationImportState
 
 NDJSON_MIME_TYPE = "application/x-ndjson"
@@ -91,7 +93,7 @@ def wait_until_done(self,
         """
         pbar = tqdm(total=100,
                     bar_format="{n}% |{bar}| [{elapsed}, {rate_fmt}{postfix}]"
-                   ) if show_progress else None
+                    ) if show_progress else None
         while self.state.value == AnnotationImportState.RUNNING.value:
             logger.info(f"Sleeping for {sleep_time_seconds} seconds...")
             time.sleep(sleep_time_seconds)
@@ -451,6 +453,13 @@ def create_from_objects(
         if not data_str:
             raise ValueError('annotations cannot be empty')
         data = data_str.encode('utf-8')
+
+        has_confidence = LabelsConfidencePresenceChecker.check(predictions)
+        if has_confidence:
+            logger.warning("""
+            Confidence scores are not supported in MAL Prediction Import. 
+            Corresponding confidence score values will be ingored.
+            """)
         return cls._create_mal_import_from_bytes(client, project_id, name, data,
                                                  len(data))
 
@@ -603,6 +612,13 @@ def create_from_objects(cls, client: "labelbox.Client", project_id: str,
         if not data_str:
             raise ValueError('labels cannot be empty')
         data = data_str.encode('utf-8')
+        
+        has_confidence = LabelsConfidencePresenceChecker.check(labels)
+        if has_confidence:
+            logger.warning("""
+            Confidence scores are not supported in Label Import. 
+            Corresponding confidence score values will be ignored.
+            """)
         return cls._create_label_import_from_bytes(client, project_id, name,
                                                    data, len(data))
 
@@ -661,7 +677,8 @@ def from_name(cls,
         }
         response = client.execute(query_str, params)
         if response is None:
-            raise labelbox.exceptions.ResourceNotFoundError(LabelImport, params)
+            raise labelbox.exceptions.ResourceNotFoundError(
+                LabelImport, params)
         response = response["labelImport"]
         if as_json:
             return response

labelbox/schema/confidence_presence_checker.py

@@ -0,0 +1,52 @@
+from typing import Any, Dict, List, Union
+
+from labelbox.data.annotation_types.annotation import (
+    ClassificationAnnotation, ObjectAnnotation, VideoClassificationAnnotation,
+    VideoObjectAnnotation)
+from labelbox.data.annotation_types.classification.classification import (
+    Checklist, ClassificationAnswer, Dropdown, Radio, Text)
+from labelbox.data.annotation_types.label import Label
+from labelbox.data.annotation_types.metrics.confusion_matrix import \
+    ConfusionMatrixMetric
+from labelbox.data.annotation_types.metrics.scalar import ScalarMetric
+from labelbox.data.serialization.ndjson.converter import NDJsonConverter
+
+
+class LabelsConfidencePresenceChecker:
+    """
+    Checks if a given list of labels contains at least one confidence score
+    """
+    @classmethod
+    def check(cls, raw_labels: List[Dict[str, Any]]):
+        label_list = NDJsonConverter.deserialize(raw_labels).as_list()
+        return any([cls._check_label(x) for x in label_list])
+
+    @classmethod
+    def _check_label(cls, label: Label):
+        return any([cls._check_annotation(x) for x in label.annotations])
+
+    @classmethod
+    def _check_annotation(cls, annotation: Union[ClassificationAnnotation, ObjectAnnotation,
+                                                 VideoObjectAnnotation,
+                                                 VideoClassificationAnnotation, ScalarMetric,
+                                                 ConfusionMatrixMetric]):
+        if annotation.confidence is not None:
+            return True
+        if annotation.classifications:
+            return any([cls._check_classification(x) for x in annotation.classifications])
+        return False
+
+    @classmethod
+    def _check_classification(cls, classification: ClassificationAnnotation) -> bool:
+        if isinstance(classification.value,    (Checklist, Dropdown)):
+            return any(cls._check_classification_answer(x) for x in classification.value.answer)
+        if isinstance(classification.value, Radio):
+            return cls._check_classification_answer(classification.value.answer)
+        if isinstance(classification.value, Text):
+            return False
+        raise Exception(
+            f"Unexpected classification value type {type(classification.value)}")
+
+    @classmethod
+    def _check_classification_answer(cls, answer: ClassificationAnswer) -> bool:
+        return answer.confidence is not None

tests/unit/test_label_import.py

@@ -0,0 +1,34 @@
+import uuid
+from unittest.mock import MagicMock, patch
+
+from labelbox.schema.annotation_import import LabelImport, logger
+
+
+def test_should_warn_user_about_unsupported_confidence():
+    """this test should check running state only to validate running, not completed"""
+    id = str(uuid.uuid4())
+
+    labels = [{
+        "uuid": "b862c586-8614-483c-b5e6-82810f70cac0",
+                "schemaId": "ckrazcueb16og0z6609jj7y3y",
+                "dataRow": {
+                    "id": "ckrazctum0z8a0ybc0b0o0g0v"
+                },
+        "confidence": 0.851,
+        "bbox": {
+                    "top": 1352,
+                    "left": 2275,
+                    "height": 350,
+                    "width": 139
+                }
+    }, ]
+    with patch.object(LabelImport, '_create_label_import_from_bytes'):
+        with patch.object(logger, 'warning') as warning_mock:
+            LabelImport.create_from_objects(
+                client=MagicMock(),
+                project_id=id,
+                name=id,
+                labels=labels)
+            warning_mock.assert_called_once()
+            "Confidence scores are not supported in Label Import" in warning_mock.call_args_list[
+                0].args[0]

tests/unit/test_mal_import.py

@@ -0,0 +1,44 @@
+import uuid
+from unittest.mock import MagicMock, patch
+
+from labelbox.schema.annotation_import import MALPredictionImport, logger
+
+
+def test_should_warn_user_about_unsupported_confidence():
+    """this test should check running state only to validate running, not completed"""
+    id = str(uuid.uuid4())
+
+    labels = [{
+        "bbox": {
+            "height": 428,
+            "left": 2089,
+            "top": 1251,
+            "width": 158
+        },
+        "classifications": [
+            {
+                "answer": [
+                    {
+                        "schemaId": "ckrb1sfl8099e0y919v260awv",
+                        "confidence": 0.894
+                    }
+                ],
+                "schemaId": "ckrb1sfkn099c0y910wbo0p1a"
+            }
+        ],
+        "dataRow": {
+            "id": "ckrb1sf1i1g7i0ybcdc6oc8ct"
+        },
+        "schemaId": "ckrb1sfjx099a0y914hl319ie",
+        "uuid": "d009925d-91a3-4f67-abd9-753453f5a584"
+    }, ]
+    with patch.object(MALPredictionImport, '_create_mal_import_from_bytes'):
+        with patch.object(logger, 'warning') as warning_mock:
+            MALPredictionImport.create_from_objects(
+                client=MagicMock(),
+                project_id=id,
+                name=id,
+                predictions=labels)
+            warning_mock.assert_called_once()
+            "Confidence scores are not supported in MAL Prediction Import" in warning_mock.call_args_list[
+                0].args[0]