Swap setting deprecations (#1497)

* Deprecate analytics_events.max_samples_stored

* Deprecate transaction_events.max_samples_stored

* Deprecate span_events.max_samples_stored

* Deprecate event_harvest_config.harvest_limits.error_event_data

* Deprecate custom_insights_events.max_samples_stored

* Deprecate application_logging.forwarding.max_samples_stored

* Add settings to core/config

* Megalinter fixes

* Revert env var to previous value

* Add variable for default value

* Use preferred settings in agent

* Megalinter fixes

* Reviewer changes

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: lrafeei

newrelic/api/transaction.py

@@ -333,13 +333,9 @@ def __init__(self, application, enabled=None, source=None):
                     self.enabled = True
 
         if self._settings:
-            self._custom_events = SampledDataSet(
-                capacity=self._settings.event_harvest_config.harvest_limits.custom_event_data
-            )
+            self._custom_events = SampledDataSet(capacity=self._settings.custom_insights_events.max_samples_stored)
             self._ml_events = SampledDataSet(capacity=self._settings.event_harvest_config.harvest_limits.ml_event_data)
-            self._log_events = SampledDataSet(
-                capacity=self._settings.event_harvest_config.harvest_limits.log_event_data
-            )
+            self._log_events = SampledDataSet(capacity=self._settings.application_logging.forwarding.max_samples_stored)
         else:
             self._custom_events = SampledDataSet(capacity=CUSTOM_EVENT_RESERVOIR_SIZE)
             self._log_events = SampledDataSet(capacity=LOG_EVENT_RESERVOIR_SIZE)

newrelic/config.py

@@ -669,12 +669,19 @@ def translate_deprecated_settings(settings, cached_settings):
     cached = dict(cached_settings)
 
     deprecated_settings_map = [
-        ("analytics_events.max_samples_stored", "event_harvest_config.harvest_limits.analytic_event_data"),
-        ("transaction_events.max_samples_stored", "event_harvest_config.harvest_limits.analytic_event_data"),
-        ("span_events.max_samples_stored", "event_harvest_config.harvest_limits.span_event_data"),
-        ("error_collector.max_event_samples_stored", "event_harvest_config.harvest_limits.error_event_data"),
-        ("custom_insights_events.max_samples_stored", "event_harvest_config.harvest_limits.custom_event_data"),
-        ("application_logging.forwarding.max_samples_stored", "event_harvest_config.harvest_limits.log_event_data"),
+        ("transaction_tracer.capture_attributes", "transaction_tracer.attributes.enabled"),
+        ("error_collector.capture_attributes", "error_collector.attributes.enabled"),
+        ("browser_monitoring.capture_attributes", "browser_monitoring.attributes.enabled"),
+        ("analytics_events.capture_attributes", "transaction_events.attributes.enabled"),
+        ("analytics_events.enabled", "transaction_events.enabled"),
+        ("analytics_events.max_samples_stored", "transaction_events.max_samples_stored"),
+        ("event_harvest_config.harvest_limits.analytic_event_data", "transaction_events.max_samples_stored"),
+        ("event_harvest_config.harvest_limits.span_event_data", "span_events.max_samples_stored"),
+        ("event_harvest_config.harvest_limits.error_event_data", "error_collector.max_event_samples_stored"),
+        ("event_harvest_config.harvest_limits.custom_event_data", "custom_insights_events.max_samples_stored"),
+        ("event_harvest_config.harvest_limits.log_event_data", "application_logging.forwarding.max_samples_stored"),
+        ("error_collector.ignore_errors", "error_collector.ignore_classes"),
+        ("strip_exception_messages.whitelist", "strip_exception_messages.allowlist"),
     ]
 
     for old_key, new_key in deprecated_settings_map:

newrelic/core/config.py

@@ -818,11 +818,17 @@ def default_otlp_host(host):
 )
 
 _settings.transaction_events.enabled = True
+_settings.transaction_events.max_samples_stored = _environ_as_int(
+    "NEW_RELIC_ANALYTICS_EVENTS_MAX_SAMPLES_STORED", default=DEFAULT_RESERVOIR_SIZE
+)
 _settings.transaction_events.attributes.enabled = True
 _settings.transaction_events.attributes.exclude = []
 _settings.transaction_events.attributes.include = []
 
 _settings.custom_insights_events.enabled = True
+_settings.custom_insights_events.max_samples_stored = _environ_as_int(
+    "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", default=CUSTOM_EVENT_RESERVOIR_SIZE
+)
 _settings.custom_insights_events.max_attribute_value = _environ_as_int(
     "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_ATTRIBUTE_VALUE", default=MAX_ATTRIBUTE_LENGTH
 )
@@ -838,6 +844,9 @@ def default_otlp_host(host):
 )
 _settings.distributed_tracing.exclude_newrelic_header = False
 _settings.span_events.enabled = _environ_as_bool("NEW_RELIC_SPAN_EVENTS_ENABLED", default=True)
+_settings.span_events.max_samples_stored = _environ_as_int(
+    "NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED", default=SPAN_EVENT_RESERVOIR_SIZE
+)
 _settings.span_events.attributes.enabled = True
 _settings.span_events.attributes.exclude = []
 _settings.span_events.attributes.include = []
@@ -865,6 +874,9 @@ def default_otlp_host(host):
 _settings.error_collector.ignore_classes = []
 _settings.error_collector.ignore_status_codes = _parse_status_codes("100-102 200-208 226 300-308 404", set())
 _settings.error_collector.expected_classes = []
+_settings.error_collector.max_event_samples_stored = _environ_as_int(
+    "NEW_RELIC_ERROR_COLLECTOR_MAX_EVENT_SAMPLES_STORED", default=ERROR_EVENT_RESERVOIR_SIZE
+)
 _settings.error_collector.expected_status_codes = set()
 _settings.error_collector._error_group_callback = None
 _settings.error_collector.attributes.enabled = True
@@ -1021,6 +1033,9 @@ def default_otlp_host(host):
 _settings.application_logging.forwarding.custom_attributes = _environ_as_mapping(
     "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES", default=""
 )
+_settings.application_logging.forwarding.max_samples_stored = _environ_as_int(
+    "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_MAX_SAMPLES_STORED", default=LOG_EVENT_RESERVOIR_SIZE
+)
 
 _settings.application_logging.forwarding.labels.enabled = _environ_as_bool(
     "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_ENABLED", default=False
@@ -1306,9 +1321,7 @@ def apply_server_side_settings(server_side_config=None, settings=_settings):
     span_event_harvest_config = server_side_config.get("span_event_harvest_config", {})
     span_event_harvest_limit = span_event_harvest_config.get("harvest_limit", None)
     if span_event_harvest_limit is not None:
-        apply_config_setting(
-            settings_snapshot, "event_harvest_config.harvest_limits.span_event_data", span_event_harvest_limit
-        )
+        apply_config_setting(settings_snapshot, "span_events.max_samples_stored", span_event_harvest_limit)
 
     # Check to see if collect_ai appears in the connect response to handle account-level AIM toggling
     collect_ai = server_side_config.get("collect_ai", None)

newrelic/core/stats_engine.py

@@ -1704,21 +1704,19 @@ def reset_transaction_events(self):
         """
 
         if self.__settings is not None:
-            self._transaction_events = SampledDataSet(
-                self.__settings.event_harvest_config.harvest_limits.analytic_event_data
-            )
+            self._transaction_events = SampledDataSet(self.__settings.transaction_events.max_samples_stored)
         else:
             self._transaction_events = SampledDataSet()
 
     def reset_error_events(self):
         if self.__settings is not None:
-            self._error_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.error_event_data)
+            self._error_events = SampledDataSet(self.__settings.error_collector.max_event_samples_stored)
         else:
             self._error_events = SampledDataSet()
 
     def reset_custom_events(self):
         if self.__settings is not None:
-            self._custom_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.custom_event_data)
+            self._custom_events = SampledDataSet(self.__settings.custom_insights_events.max_samples_stored)
         else:
             self._custom_events = SampledDataSet()
 
@@ -1730,13 +1728,13 @@ def reset_ml_events(self):
 
     def reset_span_events(self):
         if self.__settings is not None:
-            self._span_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.span_event_data)
+            self._span_events = SampledDataSet(self.__settings.span_events.max_samples_stored)
         else:
             self._span_events = SampledDataSet()
 
     def reset_log_events(self):
         if self.__settings is not None:
-            self._log_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.log_event_data)
+            self._log_events = SampledDataSet(self.__settings.application_logging.forwarding.max_samples_stored)
         else:
             self._log_events = SampledDataSet()
 

tests/agent_features/test_configuration.py

@@ -406,52 +406,94 @@ def test_delete_setting_parent():
 
 translate_settings_tests = [
     (
-        TSetting("analytics_events.max_samples_stored", 1200, 1200),
-        TSetting("event_harvest_config.harvest_limits.analytic_event_data", 9999, 1200),
+        TSetting("strip_exception_messages.whitelist", [], []),
+        TSetting("strip_exception_messages.allowlist", ["non-default-value"], []),
     ),
     (
-        TSetting("analytics_events.max_samples_stored", 9999, 1200),
-        TSetting("event_harvest_config.harvest_limits.analytic_event_data", 1200, 1200),
+        TSetting("strip_exception_messages.whitelist", ["non-default-value"], []),
+        TSetting("strip_exception_messages.allowlist", [], []),
     ),
     (
-        TSetting("transaction_events.max_samples_stored", 1200, 1200),
-        TSetting("event_harvest_config.harvest_limits.analytic_event_data", 9999, 1200),
+        TSetting("transaction_tracer.capture_attributes", True, True),
+        TSetting("transaction_tracer.attributes.enabled", False, True),
+    ),
+    (
+        TSetting("transaction_tracer.capture_attributes", False, True),
+        TSetting("transaction_tracer.attributes.enabled", True, True),
+    ),
+    (
+        TSetting("error_collector.capture_attributes", True, True),
+        TSetting("error_collector.attributes.enabled", False, True),
+    ),
+    (
+        TSetting("error_collector.capture_attributes", False, True),
+        TSetting("error_collector.attributes.enabled", True, True),
+    ),
+    (
+        TSetting("browser_monitoring.capture_attributes", False, False),
+        TSetting("browser_monitoring.attributes.enabled", True, False),
     ),
     (
+        TSetting("browser_monitoring.capture_attributes", True, False),
+        TSetting("browser_monitoring.attributes.enabled", False, False),
+    ),
+    (
+        TSetting("analytics_events.capture_attributes", True, True),
+        TSetting("transaction_events.attributes.enabled", False, True),
+    ),
+    (
+        TSetting("analytics_events.capture_attributes", False, True),
+        TSetting("transaction_events.attributes.enabled", True, True),
+    ),
+    (TSetting("analytics_events.enabled", True, True), TSetting("transaction_events.enabled", False, True)),
+    (TSetting("analytics_events.enabled", False, True), TSetting("transaction_events.enabled", True, True)),
+    (
+        TSetting("analytics_events.max_samples_stored", 1200, 1200),
         TSetting("transaction_events.max_samples_stored", 9999, 1200),
+    ),
+    (
+        TSetting("analytics_events.max_samples_stored", 9999, 1200),
+        TSetting("transaction_events.max_samples_stored", 1200, 1200),
+    ),
+    (
         TSetting("event_harvest_config.harvest_limits.analytic_event_data", 1200, 1200),
+        TSetting("transaction_events.max_samples_stored", 9999, 1200),
     ),
     (
-        TSetting("span_events.max_samples_stored", 1000, 2000),
-        TSetting("event_harvest_config.harvest_limits.span_event_data", 9999, 2000),
+        TSetting("event_harvest_config.harvest_limits.analytic_event_data", 9999, 1200),
+        TSetting("transaction_events.max_samples_stored", 1200, 1200),
     ),
     (
-        TSetting("span_events.max_samples_stored", 9999, 2000),
         TSetting("event_harvest_config.harvest_limits.span_event_data", 1000, 2000),
+        TSetting("span_events.max_samples_stored", 9999, 2000),
     ),
     (
-        TSetting("error_collector.max_event_samples_stored", 100, 100),
-        TSetting("event_harvest_config.harvest_limits.error_event_data", 9999, 100),
+        TSetting("event_harvest_config.harvest_limits.span_event_data", 9999, 2000),
+        TSetting("span_events.max_samples_stored", 1000, 2000),
     ),
     (
-        TSetting("error_collector.max_event_samples_stored", 9999, 100),
         TSetting("event_harvest_config.harvest_limits.error_event_data", 100, 100),
+        TSetting("error_collector.max_event_samples_stored", 9999, 100),
     ),
     (
-        TSetting("custom_insights_events.max_samples_stored", 3600, 3600),
-        TSetting("event_harvest_config.harvest_limits.custom_event_data", 9999, 3600),
+        TSetting("event_harvest_config.harvest_limits.error_event_data", 9999, 100),
+        TSetting("error_collector.max_event_samples_stored", 100, 100),
     ),
     (
-        TSetting("custom_insights_events.max_samples_stored", 9999, 3600),
         TSetting("event_harvest_config.harvest_limits.custom_event_data", 3600, 3600),
+        TSetting("custom_insights_events.max_samples_stored", 9999, 3600),
     ),
     (
-        TSetting("application_logging.forwarding.max_samples_stored", 10000, 10000),
-        TSetting("event_harvest_config.harvest_limits.log_event_data", 99999, 10000),
+        TSetting("event_harvest_config.harvest_limits.custom_event_data", 9999, 3600),
+        TSetting("custom_insights_events.max_samples_stored", 3600, 3600),
     ),
     (
-        TSetting("application_logging.forwarding.max_samples_stored", 99999, 10000),
         TSetting("event_harvest_config.harvest_limits.log_event_data", 10000, 10000),
+        TSetting("application_logging.forwarding.max_samples_stored", 99999, 10000),
+    ),
+    (
+        TSetting("event_harvest_config.harvest_limits.log_event_data", 99999, 10000),
+        TSetting("application_logging.forwarding.max_samples_stored", 10000, 10000),
     ),
 ]
 

tests/agent_features/test_notice_error.py

@@ -411,7 +411,7 @@ def test_transaction_error_event_limit():
 @override_application_settings(
     {
         "agent_limits.errors_per_harvest": _errors_per_harvest_limit,
-        "event_harvest_config.harvest_limits.error_event_data": _error_event_limit,
+        "error_collector.max_event_samples_stored": _error_event_limit,
     }
 )
 @reset_core_stats_engine()

tests/agent_features/test_priority_sampling.py

@@ -24,7 +24,7 @@
 from newrelic.api.background_task import BackgroundTask
 
 
-@override_application_settings({"event_harvest_config.harvest_limits.analytic_event_data": 1})
+@override_application_settings({"transaction_events.max_samples_stored": 1})
 @pytest.mark.parametrize("first_transaction_saved", [True, False])
 def test_priority_used_in_transaction_events(first_transaction_saved):
     first_priority = 1 if first_transaction_saved else 0
@@ -57,7 +57,7 @@ def _test():
     _test()
 
 
-@override_application_settings({"event_harvest_config.harvest_limits.error_event_data": 1})
+@override_application_settings({"error_collector.max_event_samples_stored": 1})
 @pytest.mark.parametrize("first_transaction_saved", [True, False])
 def test_priority_used_in_transaction_error_events(first_transaction_saved):
     first_priority = 1 if first_transaction_saved else 0
@@ -97,7 +97,7 @@ def _test():
     _test()
 
 
-@override_application_settings({"event_harvest_config.harvest_limits.custom_event_data": 1})
+@override_application_settings({"custom_insights_events.max_samples_stored": 1})
 @pytest.mark.parametrize("first_transaction_saved", [True, False])
 def test_priority_used_in_transaction_custom_events(first_transaction_saved):
     first_priority = 1 if first_transaction_saved else 0

tests/agent_unittests/test_connect_response_fields.py

@@ -140,7 +140,7 @@ def test_span_event_harvest_config(connect_response_fields):
         from newrelic.core.config import SPAN_EVENT_RESERVOIR_SIZE
 
         expected = SPAN_EVENT_RESERVOIR_SIZE
-    assert protocol.configuration.event_harvest_config.harvest_limits.span_event_data == expected
+    assert protocol.configuration.span_events.max_samples_stored == expected
 
 
 @override_generic_settings(global_settings(), {"developer_mode": True})

tests/agent_unittests/test_harvest_loop.py

@@ -349,7 +349,7 @@ def test_application_harvest_with_spans(distributed_tracing_enabled, span_events
             "license_key": "**NOT A LICENSE KEY**",
             "distributed_tracing.enabled": distributed_tracing_enabled,
             "span_events.enabled": span_events_enabled,
-            "event_harvest_config.harvest_limits.span_event_data": max_samples_stored,
+            "span_events.max_samples_stored": max_samples_stored,
         },
     )
     def _test():
@@ -514,10 +514,10 @@ def test_adaptive_sampling(transaction_node, monkeypatch):
         "feature_flag": set(),
         "distributed_tracing.enabled": True,
         "application_logging.forwarding.enabled": True,
-        "event_harvest_config.harvest_limits.error_event_data": 1000,
-        "event_harvest_config.harvest_limits.span_event_data": 1000,
-        "event_harvest_config.harvest_limits.custom_event_data": 1000,
-        "event_harvest_config.harvest_limits.log_event_data": 1000,
+        "error_collector.max_event_samples_stored": 1000,
+        "span_events.max_samples_stored": 1000,
+        "custom_insights_events.max_samples_stored": 1000,
+        "application_logging.forwarding.max_samples_stored": 1000,
     },
 )
 def test_reservoir_sizes(transaction_node):
@@ -537,13 +537,13 @@ def test_reservoir_sizes(transaction_node):
 
 
 @pytest.mark.parametrize(
-    "harvest_name, event_name",
+    "harvest_setting,event_name",
     [
-        ("analytic_event_data", "transaction_events"),
-        ("error_event_data", "error_events"),
-        ("custom_event_data", "custom_events"),
-        ("log_event_data", "log_events"),
-        ("span_event_data", "span_events"),
+        ("transaction_events.max_samples_stored", "transaction_events"),
+        ("error_collector.max_event_samples_stored", "error_events"),
+        ("custom_insights_events.max_samples_stored", "custom_events"),
+        ("application_logging.forwarding.max_samples_stored", "log_events"),
+        ("span_events.max_samples_stored", "span_events"),
     ],
 )
 @override_generic_settings(
@@ -555,11 +555,18 @@ def test_reservoir_sizes(transaction_node):
         "distributed_tracing.enabled": True,
     },
 )
-def test_reservoir_size_zeros(harvest_name, event_name):
+def test_reservoir_size_zeros(harvest_setting, event_name):
     app = Application("Python Agent Test (Harvest Loop)")
     app.connect_to_data_collector(None)
 
-    setattr(settings.event_harvest_config.harvest_limits, harvest_name, 0)
+    # Walk down the settings tree until the 2nd to last setting name is reached to get the
+    # settings container, then set the final setting on that container to 0
+    harvest_setting = list(harvest_setting.split("."))
+    _settings = settings
+    for setting_attr in harvest_setting[:-1]:
+        _settings = getattr(_settings, setting_attr)
+    setattr(_settings, harvest_setting[-1], 0)
+
     settings.event_harvest_config.allowlist = frozenset(())
     app._stats_engine.reset_stats(settings)
 
@@ -607,7 +614,7 @@ def test_error_event_sampling_info(events_seen):
         {
             "developer_mode": True,
             "license_key": "**NOT A LICENSE KEY**",
-            "event_harvest_config.harvest_limits.error_event_data": reservoir_size,
+            "error_collector.max_event_samples_stored": reservoir_size,
         },
     )
     def _test():
@@ -676,7 +683,7 @@ def transactions_validator(payload):
         settings,
         {
             "developer_mode": True,
-            "event_harvest_config.harvest_limits.analytic_event_data": transactions_limit,
+            "transaction_events.max_samples_stored": transactions_limit,
             "agent_limits.synthetics_events": synthetics_limit,
         },
     )

tests/logger_logging/conftest.py

@@ -31,7 +31,7 @@
     "application_logging.forwarding.context_data.enabled": True,
     "application_logging.metrics.enabled": True,
     "application_logging.local_decorating.enabled": True,
-    "event_harvest_config.harvest_limits.log_event_data": 100000,
+    "application_logging.forwarding.max_samples_stored": 100000,
 }
 
 collector_agent_registration = collector_agent_registration_fixture(