refactor: cut test execution time (#305)

This PR decreases test execution time for the test_client test suite by about 2 whole minutes (132 seconds, I think).

It does that by extracting the `aggregate_metrics` function and, instead of waiting for the client to actually send metrics, just checking what the metrics would be aggregated to.

In doing so, it might be worth adding another test to ensure that metrics **are** actually sent, but we may or may not already have this elsewhere.

## Type of change

Please delete options that are not relevant.
- [x] Refactor of tests
# How Has This Been Tested?

Please describe the tests that you ran to verify your changes.

- [x] Unit tests

# Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [ ] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream modules

Author: thomasheartman

UnleashClient/periodic_tasks/__init__.py

@@ -1,3 +1,3 @@
 # ruff: noqa: F401
 from .fetch_and_load import fetch_and_load_features
-from .send_metrics import aggregate_and_send_metrics
+from .send_metrics import aggregate_and_send_metrics, aggregate_metrics

UnleashClient/periodic_tasks/send_metrics.py

@@ -7,16 +7,9 @@
 from UnleashClient.utils import LOGGER
 
 
-def aggregate_and_send_metrics(
-    url: str,
-    app_name: str,
-    instance_id: str,
-    custom_headers: dict,
-    custom_options: dict,
+def aggregate_metrics(
     features: dict,
-    cache: BaseCache,
-    request_timeout: int,
-) -> None:
+) -> dict:
     feature_stats_list = []
 
     for feature_name in features.keys():
@@ -31,20 +24,37 @@ def aggregate_and_send_metrics(
             }
         }
 
-        features[feature_name].reset_stats()
         feature_stats_list.append(feature_stats)
 
+    return dict(ChainMap(*feature_stats_list))
+
+
+def aggregate_and_send_metrics(
+    url: str,
+    app_name: str,
+    instance_id: str,
+    custom_headers: dict,
+    custom_options: dict,
+    features: dict,
+    cache: BaseCache,
+    request_timeout: int,
+) -> None:
+    feature_stats_dict = aggregate_metrics(features)
+
+    for feature_name in features.keys():
+        features[feature_name].reset_stats()
+
     metrics_request = {
         "appName": app_name,
         "instanceId": instance_id,
         "bucket": {
             "start": cache.get(METRIC_LAST_SENT_TIME).isoformat(),
             "stop": datetime.now(timezone.utc).isoformat(),
-            "toggles": dict(ChainMap(*feature_stats_list)),
+            "toggles": feature_stats_dict,
         },
     }
 
-    if feature_stats_list:
+    if feature_stats_dict:
         send_metrics(
             url, metrics_request, custom_headers, custom_options, request_timeout
         )

tests/unit_tests/test_client.py

@@ -1,4 +1,3 @@
-import json
 import time
 import warnings
 from pathlib import Path
@@ -39,6 +38,7 @@
 from UnleashClient.cache import FileCache
 from UnleashClient.constants import FEATURES_URL, METRICS_URL, REGISTER_URL
 from UnleashClient.events import UnleashEvent, UnleashEventType
+from UnleashClient.periodic_tasks import aggregate_metrics
 from UnleashClient.strategies import Strategy
 from UnleashClient.utils import InstanceAllowType
 
@@ -210,7 +210,7 @@ def test_uc_lifecycle(unleash_client):
         status=304,
         headers={"etag": ETAG_VALUE},
     )
-    time.sleep(16)
+    time.sleep(REFRESH_INTERVAL + 1)
 
     # Simulate server provisioning change
     responses.add(
@@ -220,7 +220,7 @@ def test_uc_lifecycle(unleash_client):
         status=200,
         headers={"etag": "W/somethingelse"},
     )
-    time.sleep(30)
+    time.sleep(REFRESH_INTERVAL * 2)
     assert len(unleash_client.features) >= 9
 
 
@@ -329,13 +329,13 @@ def test_uc_dirty_cache(unleash_client_nodestroy):
 
     # Create Unleash client and check initial load
     unleash_client.initialize_client()
-    time.sleep(5)
+    time.sleep(1)
     assert unleash_client.is_enabled("testFlag")
     unleash_client.unleash_scheduler.shutdown()
 
     # Check that everything works if previous cache exists.
     unleash_client.initialize_client()
-    time.sleep(5)
+    time.sleep(1)
     assert unleash_client.is_enabled("testFlag")
 
 
@@ -484,9 +484,8 @@ def test_uc_metrics(unleash_client):
     time.sleep(1)
     assert unleash_client.is_enabled("testFlag")
 
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"]["testFlag"]["yes"] == 1
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics["testFlag"]["yes"] == 1
 
 
 @responses.activate
@@ -496,7 +495,6 @@ def test_uc_registers_metrics_for_nonexistent_features(unleash_client):
     responses.add(
         responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200
     )
-    responses.add(responses.POST, URL + METRICS_URL, json={}, status=202)
 
     # Create Unleash client and check initial load
     unleash_client.initialize_client()
@@ -506,9 +504,8 @@ def test_uc_registers_metrics_for_nonexistent_features(unleash_client):
     unleash_client.is_enabled("nonexistent-flag")
 
     # Verify that the metrics are serialized
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"]["nonexistent-flag"]["no"] == 1
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics["nonexistent-flag"]["no"] == 1
 
 
 @responses.activate
@@ -520,16 +517,14 @@ def test_uc_metrics_dependencies(unleash_client):
         json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE,
         status=200,
     )
-    responses.add(responses.POST, URL + METRICS_URL, json={}, status=202)
 
     unleash_client.initialize_client()
     time.sleep(1)
     assert unleash_client.is_enabled("Child")
 
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"]["Child"]["yes"] == 1
-    assert "Parent" not in request["bucket"]["toggles"]
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics["Child"]["yes"] == 1
+    assert "Parent" not in metrics
 
 
 @responses.activate
@@ -539,7 +534,6 @@ def test_uc_registers_variant_metrics_for_nonexistent_features(unleash_client):
     responses.add(
         responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200
     )
-    responses.add(responses.POST, URL + METRICS_URL, json={}, status=202)
 
     # Create Unleash client and check initial load
     unleash_client.initialize_client()
@@ -548,11 +542,9 @@ def test_uc_registers_variant_metrics_for_nonexistent_features(unleash_client):
     # Check a flag that doesn't exist
     unleash_client.get_variant("nonexistent-flag")
 
-    # Verify that the metrics are serialized
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"]["nonexistent-flag"]["no"] == 1
-    assert request["bucket"]["toggles"]["nonexistent-flag"]["variants"]["disabled"] == 1
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics["nonexistent-flag"]["no"] == 1
+    assert metrics["nonexistent-flag"]["variants"]["disabled"] == 1
 
 
 @responses.activate
@@ -565,7 +557,6 @@ def test_uc_doesnt_count_metrics_for_dependency_parents(unleash_client):
         json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE,
         status=200,
     )
-    responses.add(responses.POST, URL + METRICS_URL, json={}, status=202)
 
     # Create Unleash client and check initial load
     unleash_client.initialize_client()
@@ -578,11 +569,10 @@ def test_uc_doesnt_count_metrics_for_dependency_parents(unleash_client):
     unleash_client.get_variant(child)
 
     # Verify that the parent doesn't have any metrics registered
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"][child]["yes"] == 2
-    assert request["bucket"]["toggles"][child]["variants"]["childVariant"] == 1
-    assert parent not in request["bucket"]["toggles"]
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics[child]["yes"] == 2
+    assert metrics[child]["variants"]["childVariant"] == 1
+    assert parent not in metrics
 
 
 @responses.activate
@@ -595,7 +585,6 @@ def test_uc_counts_metrics_for_child_even_if_parent_is_disabled(unleash_client):
         json=MOCK_FEATURE_WITH_DEPENDENCIES_RESPONSE,
         status=200,
     )
-    responses.add(responses.POST, URL + METRICS_URL, json={}, status=202)
 
     # Create Unleash client and check initial load
     unleash_client.initialize_client()
@@ -608,11 +597,10 @@ def test_uc_counts_metrics_for_child_even_if_parent_is_disabled(unleash_client):
     unleash_client.get_variant(child)
 
     # Verify that the parent doesn't have any metrics registered
-    time.sleep(12)
-    request = json.loads(responses.calls[-1].request.body)
-    assert request["bucket"]["toggles"][child]["no"] == 2
-    assert request["bucket"]["toggles"][child]["variants"]["disabled"] == 1
-    assert parent not in request["bucket"]["toggles"]
+    metrics = aggregate_metrics(unleash_client.features)
+    assert metrics[child]["no"] == 2
+    assert metrics[child]["variants"]["disabled"] == 1
+    assert parent not in metrics
 
 
 @responses.activate
@@ -627,7 +615,7 @@ def test_uc_disabled_registration(unleash_client_toggle_only):
 
     unleash_client.initialize_client()
     unleash_client.is_enabled("testFlag")
-    time.sleep(20)
+    time.sleep(REFRESH_INTERVAL * 2)
     assert unleash_client.is_enabled("testFlag")
 
     for api_call in responses.calls:
@@ -650,7 +638,7 @@ def test_uc_server_error(unleash_client):
     responses.add(
         responses.GET, URL + FEATURES_URL, json=MOCK_FEATURE_RESPONSE, status=200
     )
-    time.sleep(20)
+    time.sleep(REFRESH_INTERVAL * 2)
     assert unleash_client.is_enabled("testFlag")
 
 

tests/utilities/testing_constants.py

@@ -14,9 +14,9 @@
 APP_NAME = "pytest"
 ENVIRONMENT = "unit"
 INSTANCE_ID = "123"
-REFRESH_INTERVAL = 15
+REFRESH_INTERVAL = 3
 REFRESH_JITTER = None
-METRICS_INTERVAL = 10
+METRICS_INTERVAL = 2
 METRICS_JITTER = None
 DISABLE_METRICS = True
 DISABLE_REGISTRATION = True