AL-4149: Validate metric names against reserved names

Author: dubininsergey

labelbox/data/annotation_types/metrics/scalar.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional, Union
 from enum import Enum
 
-from pydantic import confloat
+from pydantic import confloat, validator
 
 from .base import ConfidenceValue, BaseMetric
 
@@ -16,6 +16,18 @@ class ScalarMetricAggregation(Enum):
     SUM = "SUM"
 
 
+RESERVED_METRIC_NAMES = (
+    'true_positive_count',
+    'false_positive_count',
+    'true_negative_count',
+    'false_negative_count',
+    'precision',
+    'recall',
+    'f1',
+    'iou'
+)
+
+
 class ScalarMetric(BaseMetric):
     """ Class representing scalar metrics
 
@@ -28,6 +40,14 @@ class ScalarMetric(BaseMetric):
     value: Union[ScalarMetricValue, ScalarMetricConfidenceValue]
     aggregation: ScalarMetricAggregation = ScalarMetricAggregation.ARITHMETIC_MEAN
 
+    @validator('metric_name')
+    def validate_metric_name(cls, name: str):
+        clean_name = name.lower().strip()
+        if name.lower().strip() in RESERVED_METRIC_NAMES:
+            raise ValueError(f"`{clean_name}` is a reserved metric name. "
+                             "Please provide another value for `metric_name`.")
+        return clean_name
+
     def dict(self, *args, **kwargs):
         res = super().dict(*args, **kwargs)
         if res.get('metric_name') is None:

labelbox/data/metrics/iou/iou.py

@@ -31,7 +31,7 @@ def miou_metric(ground_truths: List[Union[ObjectAnnotation,
     # If both gt and preds are empty there is no metric
     if iou is None:
         return []
-    return [ScalarMetric(metric_name="iou", value=iou)]
+    return [ScalarMetric(metric_name="custom_iou", value=iou)]
 
 
 def feature_miou_metric(ground_truths: List[Union[ObjectAnnotation,
@@ -62,7 +62,7 @@ def feature_miou_metric(ground_truths: List[Union[ObjectAnnotation,
         if value is None:
             continue
         metrics.append(
-            ScalarMetric(metric_name="iou", feature_name=key, value=value))
+            ScalarMetric(metric_name="custom_iou", feature_name=key, value=value))
     return metrics
 
 

tests/data/annotation_types/test_metrics.py

@@ -5,6 +5,7 @@
 from labelbox.data.annotation_types.metrics import ConfusionMatrixMetric, ScalarMetric
 from labelbox.data.annotation_types.collection import LabelList
 from labelbox.data.annotation_types import ScalarMetric, Label, ImageData
+from labelbox.data.annotation_types.metrics.scalar import RESERVED_METRIC_NAMES
 
 
 def test_legacy_scalar_metric():
@@ -56,7 +57,7 @@ def test_legacy_scalar_metric():
 ])
 def test_custom_scalar_metric(feature_name, subclass_name, aggregation, value):
     kwargs = {'aggregation': aggregation} if aggregation is not None else {}
-    metric = ScalarMetric(metric_name="iou",
+    metric = ScalarMetric(metric_name="custom_iou",
                           value=value,
                           feature_name=feature_name,
                           subclass_name=subclass_name,
@@ -80,7 +81,7 @@ def test_custom_scalar_metric(feature_name, subclass_name, aggregation, value):
             'value':
                 value,
             'metric_name':
-                'iou',
+                'custom_iou',
             **({
                 'feature_name': feature_name
             } if feature_name else {}),
@@ -192,3 +193,10 @@ def test_invalid_number_of_confidence_scores():
             metric_name="too many scores",
             value={i / 20.: [0, 1, 2, 3] for i in range(20)})
     assert "Number of confidence scores must be greater" in str(exc_info.value)
+
+
+@pytest.mark.parametrize("metric_name", RESERVED_METRIC_NAMES)
+def test_reserved_names(metric_name: str):
+    with pytest.raises(ValidationError) as exc_info:
+        ScalarMetric(metric_name=metric_name, value=0.5)
+    assert 'is a reserved metric name' in exc_info.value.errors()[0]['msg']