fix(metrics): flush upon a single metric 100th data point (#1046)

Author: knightjoel

aws_lambda_powertools/metrics/base.py

@@ -127,7 +127,7 @@ def add_metric(self, name: str, unit: Union[MetricUnit, str], value: float) -> N
         logger.debug(f"Adding metric: {name} with {metric}")
         self.metric_set[name] = metric
 
-        if len(self.metric_set) == MAX_METRICS:
+        if len(self.metric_set) == MAX_METRICS or len(metric["Value"]) == MAX_METRICS:
             logger.debug(f"Exceeded maximum of {MAX_METRICS} metrics - Publishing existing metric set")
             metrics = self.serialize_metric_set()
             print(json.dumps(metrics))

tests/functional/test_metrics.py

@@ -82,6 +82,11 @@ def a_hundred_metrics() -> List[Dict[str, str]]:
     return [{"name": f"metric_{i}", "unit": "Count", "value": 1} for i in range(100)]
 
 
+@pytest.fixture
+def a_hundred_metric_values() -> List[Dict[str, str]]:
+    return [{"name": "metric", "unit": "Count", "value": i} for i in range(100)]
+
+
 def serialize_metrics(
     metrics: List[Dict], dimensions: List[Dict], namespace: str, metadatas: List[Dict] = None
 ) -> Dict:
@@ -229,6 +234,37 @@ def test_metrics_spillover(monkeypatch, capsys, metric, dimension, namespace, a_
     assert serialized_101th_metric == expected_101th_metric
 
 
+def test_metric_values_spillover(monkeypatch, capsys, dimension, namespace, a_hundred_metric_values):
+    # GIVEN Metrics is initialized and we have over a hundred metric values to add
+    my_metrics = Metrics(namespace=namespace)
+    my_metrics.add_dimension(**dimension)
+    metric = a_hundred_metric_values[0]
+
+    # WHEN we add 100 metric values
+    for _metric in a_hundred_metric_values:
+        my_metrics.add_metric(**_metric)
+
+    # THEN it should serialize and flush the metric at the 100th value
+    # and clear all metrics and dimensions from memory
+    output = capture_metrics_output(capsys)
+    spillover_values = output[metric["name"]]
+    assert my_metrics.metric_set == {}
+    assert len(spillover_values) == 100
+
+    # GIVEN we add the 101st metric
+    # WHEN we already had a Metric class instance
+    # with an existing dimension set from the previous 100th metric batch
+    my_metrics.add_metric(**metric)
+
+    # THEN serializing the 101st value should
+    # create a new EMF object with a single value in it (101st)
+    # and contain the same dimension we previously added
+    serialized_101st_metric = my_metrics.serialize_metric_set()
+    expected_101st_metric = serialize_single_metric(metric=metric, dimension=dimension, namespace=namespace)
+    remove_timestamp(metrics=[serialized_101st_metric, expected_101st_metric])
+    assert serialized_101st_metric == expected_101st_metric
+
+
 def test_log_metrics_decorator_call_decorated_function(metric, namespace, service):
     # GIVEN Metrics is initialized
     my_metrics = Metrics(service=service, namespace=namespace)