updates to code and test cases and fix some deprecation

Author: jtsodapop

labelbox/data/metrics/group.py

@@ -4,7 +4,8 @@
 from collections import defaultdict
 from typing import Dict, List, Tuple, Union
 
-from labelbox.data.annotation_types.annotation import ClassificationAnnotation, Checklist, Radio
+from labelbox.data.annotation_types.annotation import ClassificationAnnotation, Checklist, Radio, Text
+from labelbox.data.annotation_types.classification.classification import ClassificationAnswer
 try:
     from typing import Literal
 except ImportError:
@@ -60,7 +61,13 @@ def all_have_key(features: List[FeatureSchema]) -> Tuple[bool, bool]:
     for feature in features:
         if isinstance(feature, ClassificationAnnotation):
             if isinstance(feature.value, Checklist):
-                all_names, all_schemas = all_have_key(feature.value.answer)
+                all_schemas, all_names = all_have_key(feature.value.answer)
+            #this code should be able to be refactored better
+            elif isinstance(feature.value, Text):
+                if feature.name is None:
+                    all_names = False
+                if feature.feature_schema_id is None:
+                    all_schemas = False
             else:
                 if feature.value.answer.name is None:
                     all_names = False
@@ -169,8 +176,17 @@ def _create_feature_lookup(features: List[FeatureSchema],
             #checklists
             if isinstance(feature.value, Checklist):
                 for answer in feature.value.answer:
-                    new_feature = Radio(answer=answer)
-                    grouped_features[getattr(answer, key)] = new_feature
+                    new_answer = Radio(answer=answer)
+                    new_annotation = ClassificationAnnotation(
+                        value=new_answer,
+                        name=answer.name,
+                        feature_schema_id=answer.feature_schema_id)
+
+                    grouped_features[getattr(answer,
+                                             key)].append(new_annotation)
+            #likely can be refactored
+            elif isinstance(feature.value, Text):
+                grouped_features[getattr(feature, key)].append(feature)
             else:
                 grouped_features[getattr(feature.value.answer,
                                          key)].append(feature)

tests/data/metrics/confusion_matrix/conftest.py

@@ -183,24 +183,27 @@ def radio_pairs():
     return [
         NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
                   ground_truths=[get_radio("is_animal", answer_name="yes")],
-                  expected={'is_animal': [1, 0, 0, 0]}),
+                  expected={'yes': [1, 0, 0, 0]}),
         NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
                   ground_truths=[get_radio("is_animal", answer_name="no")],
-                  expected={'is_animal': [0, 1, 0, 1]}),
+                  expected={
+                      'no': [0, 0, 0, 1],
+                      'yes': [0, 1, 0, 0]
+                  }),
         NameSpace(predictions=[get_radio("is_animal", answer_name="yes")],
                   ground_truths=[],
-                  expected={'is_animal': [0, 1, 0, 0]}),
+                  expected={'yes': [0, 1, 0, 0]}),
         NameSpace(predictions=[],
                   ground_truths=[get_radio("is_animal", answer_name="yes")],
-                  expected={'is_animal': [0, 0, 0, 1]}),
+                  expected={'yes': [0, 0, 0, 1]}),
         NameSpace(predictions=[
             get_radio("is_animal", answer_name="yes"),
             get_radio("is_short", answer_name="no")
         ],
                   ground_truths=[get_radio("is_animal", answer_name="yes")],
                   expected={
-                      'is_animal': [1, 0, 0, 0],
-                      'is_short': [0, 1, 0, 0]
+                      'no': [0, 1, 0, 0],
+                      'yes': [1, 0, 0, 0]
                   }),
         #Not supported yet:
         # NameSpace(
@@ -221,18 +224,18 @@ def checklist_pairs():
                       get_checklist("animal_attributes",
                                     answer_names=["striped"])
                   ],
-                  expected={'animal_attributes': [1, 0, 0, 0]}),
+                  expected={'striped': [1, 0, 0, 0]}),
         NameSpace(predictions=[
             get_checklist("animal_attributes", answer_names=["striped"])
         ],
                   ground_truths=[],
-                  expected={'animal_attributes': [0, 1, 0, 0]}),
+                  expected={'striped': [0, 1, 0, 0]}),
         NameSpace(predictions=[],
                   ground_truths=[
                       get_checklist("animal_attributes",
                                     answer_names=["striped"])
                   ],
-                  expected={'animal_attributes': [0, 0, 0, 1]}),
+                  expected={'striped': [0, 0, 0, 1]}),
         NameSpace(predictions=[
             get_checklist("animal_attributes",
                           answer_names=["striped", "short"])
@@ -241,15 +244,21 @@ def checklist_pairs():
                       get_checklist("animal_attributes",
                                     answer_names=["striped"])
                   ],
-                  expected={'animal_attributes': [1, 1, 0, 0]}),
+                  expected={
+                      'short': [0, 1, 0, 0],
+                      'striped': [1, 0, 0, 0]
+                  }),
         NameSpace(predictions=[
             get_checklist("animal_attributes", answer_names=["striped"])
         ],
                   ground_truths=[
                       get_checklist("animal_attributes",
                                     answer_names=["striped", "short"])
                   ],
-                  expected={'animal_attributes': [1, 0, 0, 1]}),
+                  expected={
+                      'short': [0, 0, 0, 1],
+                      'striped': [1, 0, 0, 0]
+                  }),
         NameSpace(predictions=[
             get_checklist("animal_attributes",
                           answer_names=["striped", "short", "black"])
@@ -258,7 +267,11 @@ def checklist_pairs():
                       get_checklist("animal_attributes",
                                     answer_names=["striped", "short"])
                   ],
-                  expected={'animal_attributes': [2, 1, 0, 0]}),
+                  expected={
+                      'black': [0, 1, 0, 0],
+                      'short': [1, 0, 0, 0],
+                      'striped': [1, 0, 0, 0]
+                  }),
         NameSpace(predictions=[
             get_checklist("animal_attributes",
                           answer_names=["striped", "short", "black"]),
@@ -270,8 +283,11 @@ def checklist_pairs():
                       get_checklist("animal_name", answer_names=["pup"])
                   ],
                   expected={
-                      'animal_attributes': [2, 1, 0, 0],
-                      'animal_name': [1, 1, 0, 0]
+                      'black': [0, 1, 0, 0],
+                      'doggy': [0, 1, 0, 0],
+                      'pup': [1, 0, 0, 0],
+                      'short': [1, 0, 0, 0],
+                      'striped': [1, 0, 0, 0]
                   })
 
         #Not supported yet:

tests/data/metrics/iou/data_row/conftest.py

@@ -8,7 +8,12 @@
 
 class NameSpace(SimpleNamespace):
 
-    def __init__(self, predictions, labels, expected, classifications=None):
+    def __init__(self,
+                 predictions,
+                 labels,
+                 expected,
+                 data_row_expected=None,
+                 classifications=None):
         super(NameSpace,
               self).__init__(predictions=predictions,
                              labels={
@@ -19,7 +24,8 @@ def __init__(self, predictions, labels, expected, classifications=None):
                                      'classifications': classifications or []
                                  }
                              },
-                             expected=expected)
+                             expected=expected,
+                             data_row_expected=data_row_expected)
 
 
 @pytest.fixture
@@ -314,41 +320,45 @@ def empty_radio_prediction():
 
 @pytest.fixture
 def matching_checklist():
-    return NameSpace(labels=[],
-                     classifications=[{
-                         'featureId':
-                             '1234567890111213141516171',
-                         'schemaId':
-                             'ckppid25v0000aeyjmxfwlc7t',
-                         'uuid':
-                             '76e0dcea-fe46-43e5-95f5-a5e3f378520a',
-                         'schemaId':
-                             'ckppid25v0000aeyjmxfwlc7t',
-                         'answers': [{
-                             'schemaId': 'ckppid25v0000aeyjmxfwlc7t',
-                         }, {
-                             'schemaId': 'ckppide010001aeyj0yhiaghc'
-                         }, {
-                             'schemaId': 'ckppidq4u0002aeyjmcc4toxw'
-                         }]
-                     }],
-                     predictions=[{
-                         'uuid':
-                             '76e0dcea-fe46-43e5-95f5-a5e3f378520a',
-                         'schemaId':
-                             'ckppid25v0000aeyjmxfwlc7t',
-                         'dataRow': {
-                             'id': 'ckppihxc10005aeyjen11h7jh'
-                         },
-                         'answers': [{
-                             'schemaId': 'ckppid25v0000aeyjmxfwlc7t',
-                         }, {
-                             'schemaId': 'ckppide010001aeyj0yhiaghc'
-                         }, {
-                             'schemaId': 'ckppidq4u0002aeyjmcc4toxw'
-                         }]
-                     }],
-                     expected=1.)
+    return NameSpace(
+        labels=[],
+        classifications=[{
+            'featureId':
+                '1234567890111213141516171',
+            'schemaId':
+                'ckppid25v0000aeyjmxfwlc7t',
+            'uuid':
+                '76e0dcea-fe46-43e5-95f5-a5e3f378520a',
+            'schemaId':
+                'ckppid25v0000aeyjmxfwlc7t',
+            'answers': [{
+                'schemaId': 'ckppid25v0000aeyjmxfwlc7t',
+            }, {
+                'schemaId': 'ckppide010001aeyj0yhiaghc'
+            }, {
+                'schemaId': 'ckppidq4u0002aeyjmcc4toxw'
+            }]
+        }],
+        predictions=[{
+            'uuid':
+                '76e0dcea-fe46-43e5-95f5-a5e3f378520a',
+            'schemaId':
+                'ckppid25v0000aeyjmxfwlc7t',
+            'dataRow': {
+                'id': 'ckppihxc10005aeyjen11h7jh'
+            },
+            'answers': [{
+                'schemaId': 'ckppid25v0000aeyjmxfwlc7t',
+            }, {
+                'schemaId': 'ckppide010001aeyj0yhiaghc'
+            }, {
+                'schemaId': 'ckppidq4u0002aeyjmcc4toxw'
+            }]
+        }],
+        data_row_expected=1.,
+        #  expected = [1.]
+        #  expected=[1., 1., 1.])
+        expected={1.0: 3})
 
 
 @pytest.fixture
@@ -391,7 +401,11 @@ def partially_matching_checklist_1():
                              'schemaId': 'ckppiebx80004aeyjuwvos69e'
                          }]
                      }],
-                     expected=0.6)
+                     data_row_expected=0.6,
+                     expected={
+                         0.0: 2,
+                         1.0: 3
+                     })
 
 
 @pytest.fixture
@@ -430,7 +444,11 @@ def partially_matching_checklist_2():
                              'schemaId': 'ckppiebx80004aeyjuwvos69e'
                          }]
                      }],
-                     expected=0.5)
+                     data_row_expected=0.5,
+                     expected={
+                         1.0: 2,
+                         0.0: 2
+                     })
 
 
 @pytest.fixture
@@ -469,7 +487,11 @@ def partially_matching_checklist_3():
                              'schemaId': 'ckppide010001aeyj0yhiaghc'
                          }]
                      }],
-                     expected=0.5)
+                     data_row_expected=0.5,
+                     expected={
+                         1.0: 2,
+                         0.0: 2
+                     })
 
 
 @pytest.fixture
@@ -485,7 +507,8 @@ def empty_checklist_label():
                              'schemaId': 'ckppid25v0000aeyjmxfwlc7t'
                          }]
                      }],
-                     expected=0)
+                     data_row_expected=0.0,
+                     expected={0.0: 1})
 
 
 @pytest.fixture
@@ -502,7 +525,8 @@ def empty_checklist_prediction():
                          }]
                      }],
                      predictions=[],
-                     expected=0)
+                     data_row_expected=0.0,
+                     expected={0.0: 1})
 
 
 @pytest.fixture

tests/data/metrics/iou/data_row/test_data_row_iou.py

@@ -31,6 +31,37 @@ def check_iou(pair, mask=None):
     assert math.isclose(feature_ious[0].value, pair.expected)
 
 
+def check_iou_checklist(pair, mask=None):
+    """specialized test since checklists have more than one feature ious """
+    default = Label(data=ImageData(uid="ckppihxc10005aeyjen11h7jh"))
+    prediction = next(NDJsonConverter.deserialize(pair.predictions), default)
+    label = next(LBV1Converter.deserialize([pair.labels]))
+    if mask:
+        for annotation in [*prediction.annotations, *label.annotations]:
+            if isinstance(annotation.value, Mask):
+                annotation.value.mask.arr = np.frombuffer(
+                    base64.b64decode(annotation.value.mask.url.encode('utf-8')),
+                    dtype=np.uint8).reshape((32, 32, 3))
+    # print("drowmiou\n\n", data_row_miou(label, prediction),
+    #   pair.data_row_expected, "\n")
+    assert math.isclose(data_row_miou(label, prediction),
+                        pair.data_row_expected)
+    assert math.isclose(
+        miou_metric(label.annotations, prediction.annotations)[0].value,
+        pair.data_row_expected)
+    feature_ious = feature_miou_metric(label.annotations,
+                                       prediction.annotations)
+    b = [feature.value for feature in feature_ious]
+    # print("fiou\n\n", b, pair.expected, "\n")
+    mapping = {}
+    for iou in feature_ious:
+        print(iou)
+        if not mapping.get(iou.value, None):
+            mapping[iou.value] = 0
+        mapping[iou.value] += 1
+    assert mapping == pair.expected
+
+
 def strings_to_fixtures(strings):
     return [fixture_ref(x) for x in strings]
 
@@ -70,7 +101,7 @@ def test_radio(pair):
                  "empty_checklist_prediction",
              ]))
 def test_checklist(pair):
-    check_iou(pair)
+    check_iou_checklist(pair)
 
 
 @parametrize("pair", strings_to_fixtures(["matching_text",

tests/integration/annotation_import/test_ndjson_validation.py

@@ -1,6 +1,6 @@
 import pytest
 import ndjson
-from pytest_cases import pytest_parametrize_plus, fixture_ref
+from pytest_cases import parametrize, fixture_ref
 
 from labelbox.exceptions import MALValidationError
 from labelbox.schema.bulk_import_request import (NDChecklist, NDClassification,
@@ -23,14 +23,14 @@ def test_subclassification_construction(rectangle_inference):
     assert isinstance(tool.classifications[0], NDRadio)
 
 
-@pytest_parametrize_plus("inference, expected_type",
-                         [(fixture_ref('polygon_inference'), NDPolygon),
-                          (fixture_ref('rectangle_inference'), NDRectangle),
-                          (fixture_ref('line_inference'), NDPolyline),
-                          (fixture_ref('entity_inference'), NDTextEntity),
-                          (fixture_ref('segmentation_inference'), NDMask),
-                          (fixture_ref('segmentation_inference_rle'), NDMask),
-                          (fixture_ref('segmentation_inference_png'), NDMask)])
+@parametrize("inference, expected_type",
+             [(fixture_ref('polygon_inference'), NDPolygon),
+              (fixture_ref('rectangle_inference'), NDRectangle),
+              (fixture_ref('line_inference'), NDPolyline),
+              (fixture_ref('entity_inference'), NDTextEntity),
+              (fixture_ref('segmentation_inference'), NDMask),
+              (fixture_ref('segmentation_inference_rle'), NDMask),
+              (fixture_ref('segmentation_inference_png'), NDMask)])
 def test_tool_construction(inference, expected_type):
     assert isinstance(NDTool.build(inference), expected_type)