Support confidence as an attribute of Text (freetext)

Support confidence inside Text for top-level free text

Support for deserialization of confidence for free text as top node

Add free text as a classification to bbox fixture

Author: Val Brodsky

labelbox/data/serialization/ndjson/classification.py

@@ -64,12 +64,15 @@ class NDTextSubclass(NDAnswer):
     answer: str
 
     def to_common(self) -> Text:
-        return Text(answer=self.answer)
+        return Text(answer=self.answer, confidence=self.confidence)
 
     @classmethod
     def from_common(cls, text: Text, name: str,
                     feature_schema_id: Cuid) -> "NDTextSubclass":
-        return cls(answer=text.answer, name=name, schema_id=feature_schema_id)
+        return cls(answer=text.answer,
+                   name=name,
+                   schema_id=feature_schema_id,
+                   confidence=text.confidence)
 
 
 class NDChecklistSubclass(NDAnswer):
@@ -161,7 +164,7 @@ def from_common(cls,
             schema_id=feature_schema_id,
             uuid=uuid,
             message_id=message_id,
-            confidence=confidence,
+            confidence=text.confidence,
         )
 
 
@@ -273,7 +276,6 @@ def to_common(
             feature_schema_id=annotation.schema_id,
             extra={'uuid': annotation.uuid},
             message_id=annotation.message_id,
-            confidence=annotation.confidence,
         )
 
         if getattr(annotation, 'frames', None) is None:

tests/data/serialization/ndjson/test_text.py

@@ -1,5 +1,5 @@
 from labelbox.data.annotation_types.annotation import ClassificationAnnotation
-from labelbox.data.annotation_types.classification.classification import ClassificationAnswer, Radio, Text
+from labelbox.data.annotation_types.classification.classification import Checklist, ClassificationAnswer, Radio, Text
 from labelbox.data.annotation_types.data.text import TextData
 from labelbox.data.annotation_types.label import Label
 
@@ -13,40 +13,92 @@ def test_serialization():
                       text="This is a test",
                   ),
                   annotations=[
-                      ClassificationAnnotation(
-                          name="radio_question_geo",
-                          confidence=0.5,
-                          value=Text(answer="first_radio_answer"))
+                      ClassificationAnnotation(name="free_text_annotation",
+                                               value=Text(confidence=0.5,
+                                                          answer="text_answer"))
                   ])
 
     serialized = NDJsonConverter.serialize([label])
     res = next(serialized)
+
     assert res['confidence'] == 0.5
-    assert res['name'] == "radio_question_geo"
-    assert res['answer'] == "first_radio_answer"
+    assert res['name'] == "free_text_annotation"
+    assert res['answer'] == "text_answer"
     assert res['dataRow']['id'] == "bkj7z2q0b0000jx6x0q2q7q0d"
 
     deserialized = NDJsonConverter.deserialize([res])
     res = next(deserialized)
+
     annotation = res.annotations[0]
-    assert annotation.confidence == 0.5
+    answer = annotation.value.answer
 
     annotation_value = annotation.value
     assert type(annotation_value) is Text
-    assert annotation_value.answer == "first_radio_answer"
+    assert annotation_value.answer == "text_answer"
+    assert annotation_value.confidence == 0.5
+
+
+def test_nested_serialization():
+    label = Label(
+        uid="ckj7z2q0b0000jx6x0q2q7q0d",
+        data=TextData(
+            uid="bkj7z2q0b0000jx6x0q2q7q0d",
+            text="This is a test",
+        ),
+        annotations=[
+            ClassificationAnnotation(
+                name="nested test",
+                value=Checklist(answer=[
+                    ClassificationAnswer(
+                        name="first_answer",
+                        confidence=0.9,
+                        classifications=[
+                            ClassificationAnnotation(
+                                name="sub_radio_question",
+                                value=Radio(answer=ClassificationAnswer(
+                                    name="first_sub_radio_answer",
+                                    confidence=0.8,
+                                    classifications=[
+                                        ClassificationAnnotation(
+                                            name="nested answer",
+                                            value=Text(
+                                                answer="nested answer",
+                                                confidence=0.7,
+                                            ))
+                                    ])))
+                        ])
+                ]),
+            )
+        ])
 
     serialized = NDJsonConverter.serialize([label])
     res = next(serialized)
-    assert res['confidence'] == 0.5
-    assert res['name'] == "radio_question_geo"
-    assert res['answer'] == "first_radio_answer"
+
     assert res['dataRow']['id'] == "bkj7z2q0b0000jx6x0q2q7q0d"
+    answer = res['answer'][0]
+    assert answer['confidence'] == 0.9
+    assert answer['name'] == "first_answer"
+    classification = answer['classifications'][0]
+    nested_classification_answer = classification['answer']
+    assert nested_classification_answer['confidence'] == 0.8
+    assert nested_classification_answer['name'] == "first_sub_radio_answer"
+    sub_classification = nested_classification_answer['classifications'][0]
+    assert sub_classification['name'] == "nested answer"
+    assert sub_classification['answer'] == "nested answer"
+    assert sub_classification['confidence'] == 0.7
 
     deserialized = NDJsonConverter.deserialize([res])
     res = next(deserialized)
     annotation = res.annotations[0]
-    assert annotation.confidence == 0.5
+    answer = annotation.value.answer[0]
+    assert answer.confidence == 0.9
+    assert answer.name == "first_answer"
 
-    annotation_value = annotation.value
-    assert type(annotation_value) is Text
-    assert annotation_value.answer == "first_radio_answer"
+    classification_answer = answer.classifications[0].value.answer
+    assert classification_answer.confidence == 0.8
+    assert classification_answer.name == "first_sub_radio_answer"
+
+    sub_classification_answer = classification_answer.classifications[0].value
+    assert type(sub_classification_answer) is Text
+    assert sub_classification_answer.answer == "nested answer"
+    assert sub_classification_answer.confidence == 0.7

tests/integration/annotation_import/conftest.py

@@ -242,13 +242,20 @@ def ontology():
                         'checklist',
                     'options': [{
                         'label': 'nested_checkbox_option_1',
-                        'value': 'nested_checkbox_value_1'
+                        'value': 'nested_checkbox_value_1',
+                        'options': []
                     }, {
                         'label': 'nested_checkbox_option_2',
                         'value': 'nested_checkbox_value_2'
                     }]
+                }, {
+                    'required': False,
+                    'instructions': 'nested_text',
+                    'name': 'nested_text',
+                    'type': 'text',
+                    'options': []
                 }]
-            }]
+            },]
         }]
     }
 
@@ -430,6 +437,7 @@ def configured_project(client, ontology, rand_gen, image_url):
             where=LabelingFrontend.name == "editor"))[0]
     project.setup(editor, ontology)
     data_row_ids = []
+
     for _ in range(len(ontology['tools']) + len(ontology['classifications'])):
         data_row_ids.append(dataset.create_data_row(row_data=image_url).uid)
     project._wait_until_data_rows_are_processed(data_row_ids=data_row_ids)
@@ -559,14 +567,34 @@ def rectangle_inference(prediction_id_mapping):
                     ['featureSchemaId'],
                 "name":
                     rectangle['tool']['classifications'][0]['options'][0]
-                    ['value']
+                    ['value'],
+                "classifications": [{
+                    "schemaId":
+                        rectangle['tool']['classifications'][0]['options'][0]
+                        ['options'][1]['featureSchemaId'],
+                    "name":
+                        rectangle['tool']['classifications'][0]['options'][0]
+                        ['options'][1]['name'],
+                    "answer":
+                        'nested answer'
+                }],
             }
         }]
     })
     del rectangle['tool']
     return rectangle
 
 
+@pytest.fixture
+def rectangle_inference_with_confidence(rectangle_inference):
+    rectangle = rectangle_inference.copy()
+    rectangle.update({"confidence": 0.9})
+    rectangle["classifications"][0]["answer"]["confidence"] = 0.8
+    rectangle["classifications"][0]["answer"]["classifications"][0][
+        "confidence"] = 0.7
+    return rectangle
+
+
 @pytest.fixture
 def rectangle_inference_document(rectangle_inference):
     rectangle = rectangle_inference.copy()
@@ -743,6 +771,13 @@ def text_inference(prediction_id_mapping):
     return text
 
 
+@pytest.fixture
+def text_inference_with_confidence(text_inference):
+    text = text_inference.copy()
+    text.update({'confidence': 0.9})
+    return text
+
+
 @pytest.fixture
 def text_inference_index(prediction_id_mapping):
     text = prediction_id_mapping['text_index'].copy()
@@ -799,6 +834,12 @@ def predictions(object_predictions, classification_predictions):
     return object_predictions + classification_predictions
 
 
+@pytest.fixture
+def predictions_with_confidence(text_inference_with_confidence,
+                                rectangle_inference_with_confidence):
+    return [text_inference_with_confidence, rectangle_inference_with_confidence]
+
+
 @pytest.fixture
 def model(client, rand_gen, configured_project):
     ontology = configured_project.ontology()
@@ -896,6 +937,14 @@ def check_running_state(req, name, url=None):
         assert req.status_file_url is None
         assert req.state == AnnotationImportState.RUNNING
 
+    @staticmethod
+    def download_and_assert_status(status_file_url):
+        response = requests.get(status_file_url)
+        assert response.status_code == 200
+        for line in parser.loads(response.content):
+            status = line['status']
+            assert status.upper() == 'SUCCESS'
+
     @staticmethod
     def _convert_to_plain_object(obj):
         """Some Python objects e.g. tuples can't be compared with JSON serialized data, serialize to JSON and deserialize to get plain objects"""

tests/integration/annotation_import/test_mea_prediction_import.py

@@ -37,6 +37,31 @@ def test_create_from_objects(model_run_with_data_rows, object_predictions,
     annotation_import.wait_until_done()
 
 
+def test_create_from_objects_with_confidence(predictions_with_confidence,
+                                             model_run_with_data_rows,
+                                             annotation_import_test_helpers):
+    name = str(uuid.uuid4())
+
+    object_prediction_data_rows = [
+        object_prediction["dataRow"]["id"]
+        for object_prediction in predictions_with_confidence
+    ]
+    # MUST have all data rows in the model run
+    model_run_with_data_rows.upsert_data_rows(
+        data_row_ids=object_prediction_data_rows)
+
+    annotation_import = model_run_with_data_rows.add_predictions(
+        name=name, predictions=predictions_with_confidence)
+
+    assert annotation_import.model_run_id == model_run_with_data_rows.uid
+    annotation_import_test_helpers.check_running_state(annotation_import, name)
+    annotation_import_test_helpers.assert_file_content(
+        annotation_import.input_file_url, predictions_with_confidence)
+    annotation_import.wait_until_done()
+    annotation_import_test_helpers.download_and_assert_status(
+        annotation_import.status_file_url)
+
+
 def test_create_from_objects_all_project_labels(
         model_run_with_all_project_labels, object_predictions,
         annotation_import_test_helpers):