add custom scalar metric

Author: Matt Sokoloff

labelbox/data/annotation_types/__init__.py

@@ -31,4 +31,3 @@
 from .metrics import ScalarMetric
 from .metrics import CustomScalarMetric
 from .metrics import MetricAggregation
-

labelbox/data/annotation_types/label.py

@@ -22,7 +22,8 @@ class Label(BaseModel):
     data: Union[VideoData, ImageData, TextData]
     annotations: List[Union[ClassificationAnnotation, ObjectAnnotation,
                             VideoObjectAnnotation,
-                            VideoClassificationAnnotation, ScalarMetric, CustomScalarMetric]] = []
+                            VideoClassificationAnnotation, CustomScalarMetric,
+                            ScalarMetric]] = []
     extra: Dict[str, Any] = {}
 
     def object_annotations(self) -> List[ObjectAnnotation]:

labelbox/data/annotation_types/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .scalar import ScalarMetric, CustomScalarMetric
+from .aggregations import MetricAggregation

labelbox/data/annotation_types/metrics/aggregations.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class MetricAggregation(Enum):
+    ARITHMETIC_MEAN = "ARITHMETIC_MEAN"
+    GEOMETRIC_MEAN = "GEOMETRIC_MEAN"
+    HARMONIC_MEAN = "HARMONIC_MEAN"
+    SUM = "SUM"

labelbox/data/annotation_types/metrics/scalar.py

@@ -0,0 +1,18 @@
+from labelbox.data.annotation_types.metrics.aggregations import MetricAggregation
+from typing import Any, Dict, Optional
+from pydantic import BaseModel
+
+
+class ScalarMetric(BaseModel):
+    """ Class representing metrics """
+    value: float
+    extra: Dict[str, Any] = {}
+
+
+class CustomScalarMetric(BaseModel):
+    metric_name: str
+    value: float
+    feature_name: Optional[str] = None
+    subclass_name: Optional[str] = None
+    aggregation: MetricAggregation = MetricAggregation.ARITHMETIC_MEAN
+    extra: Dict[str, Any] = {}

labelbox/schema/bulk_import_request.py

@@ -635,6 +635,7 @@ def determinants(parent_cls) -> List[str]:
 
 ###### Classifications ######
 
+
 class NDText(NDBase):
     ontology_type: Literal["text"] = "text"
     answer: str = pydantic.Field(determinant=True)

tests/data/annotation_types/test_metrics.py

@@ -5,6 +5,7 @@
 
 import pytest
 
+
 def test_scalar_metric():
     value = 10
     metric = ScalarMetric(value=value)
@@ -32,17 +33,24 @@ def test_scalar_metric():
     next(LabelList([label])).dict() == expected
 
 
-@pytest.mark.parametrize(
-    'feature_name,subclass_name,aggregation',
-    [
-        ("cat", "orange" , MetricAggregation.ARITHMETIC_MEAN),
-        ("cat", None, MetricAggregation.ARITHMETIC_MEAN),
-        (None, None, MetricAggregation.ARITHMETIC_MEAN),
-        (None, None,  None),
-    ])
+@pytest.mark.parametrize('feature_name,subclass_name,aggregation', [
+    ("cat", "orange", MetricAggregation.ARITHMETIC_MEAN),
+    ("cat", None, MetricAggregation.ARITHMETIC_MEAN),
+    (None, None, MetricAggregation.ARITHMETIC_MEAN),
+    (None, None, None),
+    ("cat", "orange", MetricAggregation.ARITHMETIC_MEAN),
+    ("cat", None, MetricAggregation.HARMONIC_MEAN),
+    (None, None, MetricAggregation.GEOMETRIC_MEAN),
+    (None, None, MetricAggregation.SUM),
+])
 def test_custom_scalar_metric(feature_name, subclass_name, aggregation):
     value = 0.5
-    metric = CustomScalarMetric(metric_name = "iou", value=value, feature_name=feature_name, subclass_name = subclass_name, aggregation = aggregation)
+    kwargs = {'aggregation': aggregation} if aggregation is not None else {}
+    metric = CustomScalarMetric(metric_name="iou",
+                                value=value,
+                                feature_name=feature_name,
+                                subclass_name=subclass_name,
+                                **kwargs)
     assert metric.value == value
 
     label = Label(data=ImageData(uid="ckrmd9q8g000009mg6vej7hzg"),
@@ -57,15 +65,15 @@ def test_custom_scalar_metric(feature_name, subclass_name, aggregation):
             'arr': None
         },
         'annotations': [{
-            'value': 10.0,
-
+            'value': value,
+            'metric_name': 'iou',
+            'feature_name': feature_name,
+            'subclass_name': subclass_name,
+            'aggregation': aggregation or MetricAggregation.ARITHMETIC_MEAN,
             'extra': {}
         }],
         'extra': {},
         'uid': None
     }
     assert label.dict() == expected
     next(LabelList([label])).dict() == expected
-
-
-