From f20911e581b6b5596c307c6a266b0ec0251fdabd Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 27 Oct 2025 12:44:35 -0700 Subject: [PATCH 01/31] Add baseline instrumentation. --- newrelic/api/error_trace.py | 23 +- newrelic/api/time_trace.py | 1 - newrelic/common/llm_utils.py | 46 ++ newrelic/config.py | 7 + newrelic/hooks/datastore_elasticsearch.py | 1 - newrelic/hooks/mlmodel_strands.py | 408 ++++++++++++++++++ tests/mlmodel_strands/conftest.py | 25 +- tests/mlmodel_strands/test_simple.py | 36 -- tests/testing_support/fixtures.py | 2 +- .../validators/validate_custom_event.py | 4 +- .../validate_error_event_collector_json.py | 2 +- .../validate_transaction_error_event_count.py | 4 +- 12 files changed, 507 insertions(+), 52 deletions(-) create mode 100644 newrelic/common/llm_utils.py create mode 100644 newrelic/hooks/mlmodel_strands.py delete mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index db63c54316..c94ed34dc9 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -15,6 +15,7 @@ import functools from newrelic.api.time_trace import current_trace, notice_error +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object @@ -43,17 +44,25 @@ def __exit__(self, exc, value, tb): ) -def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None): - def wrapper(wrapped, instance, args, kwargs): - parent = current_trace() +def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None, async_wrapper=None): + def literal_wrapper(wrapped, instance, args, kwargs): + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + if not wrapper: + parent = current_trace() + if not parent: + return wrapped(*args, **kwargs) + else: + parent = None - if parent is None: - return wrapped(*args, **kwargs) + trace = ErrorTrace(ignore, expected, status_code, parent=parent) + + if wrapper: + return wrapper(wrapped, trace)(*args, **kwargs) - with ErrorTrace(ignore, expected, status_code, parent=parent): + with trace: return wrapped(*args, **kwargs) - return FunctionWrapper(wrapped, wrapper) + return FunctionWrapper(wrapped, literal_wrapper) def error_trace(ignore=None, expected=None, status_code=None): diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index fd0f62fdef..7d2ad59a8b 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,7 +361,6 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} - # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py new file mode 100644 index 0000000000..0ae8575477 --- /dev/null +++ b/newrelic/common/llm_utils.py @@ -0,0 +1,46 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +_logger = logging.getLogger(__name__) +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" + + +def _get_llm_metadata(transaction): + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": vendor_name, + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 21ce996f6c..806563a1f6 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2946,6 +2946,13 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition("strands.agent.agent", "newrelic.hooks.mlmodel_strands", "instrument_agent_agent") + _process_module_definition( + "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" + ) + + _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") + _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") _process_module_definition( "mcp.server.fastmcp.tools.tool_manager", diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 92867d1b83..1e87ddddb0 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,7 +163,6 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) - # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py new file mode 100644 index 0000000000..8bb977808f --- /dev/null +++ b/newrelic/hooks/mlmodel_strands.py @@ -0,0 +1,408 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid + +from newrelic.api.error_trace import ErrorTraceWrapper +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.core.context import ContextOf + +_logger = logging.getLogger(__name__) +STRANDS_VERSION = get_package_version("strands-agents") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." +TOOL_OUTPUT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record output of tool call. Please report this issue to New Relic Support." + + +def wrap_agent__call__(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + # Make a copy of the invocation state before we mutate it + if "invocation_state" in bound_args: + invocation_state = bound_args["invocation_state"] = dict(bound_args["invocation_state"] or {}) + + # Attempt to save the current transaction context into the invocation state dictionary + invocation_state["_nr_transaction"] = trace + except Exception: + return wrapped(*args, **kwargs) + else: + return wrapped(**bound_args) + + +async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): + # If there's already a transaction, don't propagate anything here + if current_transaction(): + return await wrapped(*args, **kwargs) + + try: + # Grab the trace context we should be running under and pass it to ContextOf + bound_args = bind_args(wrapped, args, kwargs) + invocation_state = bound_args["invocation_state"] or {} + trace = invocation_state.pop("_nr_transaction", None) + except Exception: + return await wrapped(*args, **kwargs) + + # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + if trace: + with ContextOf(trace=trace): + return await wrapped(*args, **kwargs) + else: + return await wrapped(*args, **kwargs) + + +def wrap_stream_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + func_name = callable_name(wrapped) + agent_name = getattr(instance, "name", "agent") + function_trace_name = f"{func_name}/{agent_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + agent_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_agent_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} + return proxied_return_val + + +def _record_agent_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _record_tool_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, linking_metadata): + try: + try: + tool_output = tool_results[-1]["content"][0] if tool_results else None + error = tool_results[-1]["status"] == "error" + except Exception: + tool_output = None + error = False + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_name = strands_attrs.get("tool_name", "tool") + tool_id = strands_attrs.get("tool_id", None) + run_id = strands_attrs.get("run_id", None) + tool_input = strands_attrs.get("tool_input", None) + agent_name = strands_attrs.get("agent_name", "agent") + settings = transaction.settings or global_settings() + + tool_event_dict = { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "strands", + "ingest_source": "Python", + } + # Set error flag if the status shows an error was caught, + # it will be reported further down in the instrumentation. + if error: + tool_event_dict["error"] = True + + if settings.ai_monitoring.record_content.enabled: + tool_event_dict.update({"input": tool_input}) + tool_event_dict.update({"output": tool_output}) + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def _handle_agent_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"agent_id": agent_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _handle_tool_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + tool_id = strands_attrs["tool_id"] + + # We expect this to never have any output since this is an error case, + # but if it does we will report it. + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"tool_id": tool_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata, "strands" + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def wrap_tool_executor__stream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + # Grab tool data + bound_args = bind_args(wrapped, args, kwargs) + agent_name = getattr(bound_args.get("agent"), "name", "agent") + tool_use = bound_args.get("tool_use", {}) + + run_id = tool_use.get("toolUseId", "") + tool_name = tool_use.get("name", "tool") + _input = tool_use.get("input") + tool_input = str(_input) if _input else None + tool_results = bound_args.get("tool_results", []) + + func_name = callable_name(wrapped) + function_trace_name = f"{func_name}/{tool_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/tool/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + tool_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_tool_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = { + "tool_results": tool_results, + "tool_name": tool_name, + "tool_id": tool_id, + "run_id": run_id, + "tool_input": tool_input, + "agent_name": agent_name, + } + return proxied_return_val + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped, on_stop_iteration, on_error): + super().__init__(wrapped) + self._nr_on_stop_iteration = on_stop_iteration + self._nr_on_error = on_error + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + except StopAsyncIteration: + self._nr_on_stop_iteration(self, transaction) + raise + except Exception as exc: + self._nr_on_error(self, transaction, exc) + raise + return return_val + + async def aclose(self): + return await super().aclose() + + +def wrap_ToolRegister_register_tool(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + bound_args["tool"]._tool_func = ErrorTraceWrapper(bound_args["tool"]._tool_func) + return wrapped(*args, **kwargs) + + +def instrument_agent_agent(module): + if hasattr(module, "Agent"): + if hasattr(module.Agent, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) + if hasattr(module.Agent, "invoke_async"): + wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) + if hasattr(module.Agent, "_run_loop"): + wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) + + +def instrument_tools_executors__executor(module): + if hasattr(module, "ToolExecutor"): + if hasattr(module.ToolExecutor, "_stream"): + wrap_function_wrapper(module, "ToolExecutor._stream", wrap_tool_executor__stream) + + +def instrument_tools_registry(module): + if hasattr(module, "ToolRegistry"): + if hasattr(module.ToolRegistry, "register_tool"): + wrap_function_wrapper(module, "ToolRegistry.register_tool", wrap_ToolRegister_register_tool) diff --git a/tests/mlmodel_strands/conftest.py b/tests/mlmodel_strands/conftest.py index b810161f6a..a2ad9b8dd0 100644 --- a/tests/mlmodel_strands/conftest.py +++ b/tests/mlmodel_strands/conftest.py @@ -14,6 +14,7 @@ import pytest from _mock_model_provider import MockedModelProvider +from testing_support.fixture.event_loop import event_loop as loop from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture from testing_support.ml_testing_utils import set_trace_info @@ -50,15 +51,33 @@ def single_tool_model(): @pytest.fixture -def single_tool_model_error(): +def single_tool_model_runtime_error_coro(): model = MockedModelProvider( [ { "role": "assistant", "content": [ - {"text": "Calling add_exclamation tool"}, + {"text": "Calling throw_exception_coro tool"}, + # Set arguments to an invalid type to trigger error in tool + {"toolUse": {"name": "throw_exception_coro", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def single_tool_model_runtime_error_agen(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling throw_exception_agen tool"}, # Set arguments to an invalid type to trigger error in tool - {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": 12}}}, + {"toolUse": {"name": "throw_exception_agen", "toolUseId": "123", "input": {"message": "Hello"}}}, ], }, {"role": "assistant", "content": [{"text": "Success!"}]}, diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py deleted file mode 100644 index ae24003fab..0000000000 --- a/tests/mlmodel_strands/test_simple.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from strands import Agent, tool - -from newrelic.api.background_task import background_task - - -# Example tool for testing purposes -@tool -def add_exclamation(message: str) -> str: - return f"{message}!" - - -# TODO: Remove this file once all real tests are in place - - -@background_task() -def test_simple_run_agent(set_trace_info, single_tool_model): - set_trace_info() - my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) - - response = my_agent("Run the tools.") - assert response.message["content"][0]["text"] == "Success!" - assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 3d93e06e30..540e44f70c 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -797,7 +797,7 @@ def _bind_params(transaction, *args, **kwargs): transaction = _bind_params(*args, **kwargs) error_events = transaction.error_events(instance.stats_table) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for sample in error_events: assert isinstance(sample, list) assert len(sample) == 3 diff --git a/tests/testing_support/validators/validate_custom_event.py b/tests/testing_support/validators/validate_custom_event.py index deeef7fb25..c3cf78032a 100644 --- a/tests/testing_support/validators/validate_custom_event.py +++ b/tests/testing_support/validators/validate_custom_event.py @@ -61,7 +61,9 @@ def _validate_custom_event_count(wrapped, instance, args, kwargs): raise else: stats = core_application_stats_engine(None) - assert stats.custom_events.num_samples == count + assert stats.custom_events.num_samples == count, ( + f"Expected: {count}, Got: {stats.custom_events.num_samples}. Events: {list(stats.custom_events)}" + ) return result diff --git a/tests/testing_support/validators/validate_error_event_collector_json.py b/tests/testing_support/validators/validate_error_event_collector_json.py index d1cec3a558..27ea76f3a3 100644 --- a/tests/testing_support/validators/validate_error_event_collector_json.py +++ b/tests/testing_support/validators/validate_error_event_collector_json.py @@ -52,7 +52,7 @@ def _validate_error_event_collector_json(wrapped, instance, args, kwargs): error_events = decoded_json[2] - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for event in error_events: # event is an array containing intrinsics, user-attributes, # and agent-attributes diff --git a/tests/testing_support/validators/validate_transaction_error_event_count.py b/tests/testing_support/validators/validate_transaction_error_event_count.py index b41a52330f..f5e8c0b206 100644 --- a/tests/testing_support/validators/validate_transaction_error_event_count.py +++ b/tests/testing_support/validators/validate_transaction_error_event_count.py @@ -28,7 +28,9 @@ def _validate_error_event_on_stats_engine(wrapped, instance, args, kwargs): raise else: error_events = list(instance.error_events) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, ( + f"Expected: {num_errors}, Got: {len(error_events)}. Errors: {error_events}" + ) return result From c382f72baae5dd710b3ac8374f3e6bb51d13b278 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:11:49 -0700 Subject: [PATCH 02/31] Add tool and agent instrumentation. --- newrelic/common/llm_utils.py | 21 ------------------- newrelic/config.py | 1 - newrelic/hooks/mlmodel_strands.py | 35 +++++++++++++++++++++++-------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 0ae8575477..2ec1136e6d 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - -_logger = logging.getLogger(__name__) -RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" - def _get_llm_metadata(transaction): # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events @@ -28,19 +23,3 @@ def _get_llm_metadata(transaction): return llm_metadata_dict - -def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): - try: - agent_event_dict = { - "id": agent_id, - "name": agent_name, - "span_id": linking_metadata.get("span.id"), - "trace_id": linking_metadata.get("trace.id"), - "vendor": vendor_name, - "ingest_source": "Python", - } - agent_event_dict.update(_get_llm_metadata(transaction)) - except Exception: - agent_event_dict = {} - - return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 806563a1f6..09a82e7c9a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2950,7 +2950,6 @@ def _process_module_builtin_defaults(): _process_module_definition( "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" ) - _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 8bb977808f..6ac032d8e6 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -20,7 +20,7 @@ from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace, get_trace_linking_metadata from newrelic.api.transaction import current_transaction -from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.llm_utils import _get_llm_metadata from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version @@ -67,7 +67,7 @@ async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): except Exception: return await wrapped(*args, **kwargs) - # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + # If we find a transaction to propagate, use it. Otherwise, just call wrapped. if trace: with ContextOf(trace=trace): return await wrapped(*args, **kwargs) @@ -129,7 +129,7 @@ def _record_agent_event_on_stop_iteration(self, transaction): agent_name = strands_attrs.get("agent_name", "agent") agent_id = strands_attrs.get("agent_id", None) agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -209,6 +209,7 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li if settings.ai_monitoring.record_content.enabled: tool_event_dict.update({"input": tool_input}) + # In error cases, the output will hold the error message tool_event_dict.update({"output": tool_output}) tool_event_dict.update(_get_llm_metadata(transaction)) except Exception: @@ -218,7 +219,23 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li return tool_event_dict -def _handle_agent_streaming_completion_error(self, transaction, exc): +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "strands", + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict + +def _handle_agent_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -240,7 +257,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): # Create error event agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -253,7 +270,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): self._nr_strands_attrs.clear() -def _handle_tool_streaming_completion_error(self, transaction, exc): +def _handle_tool_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -282,7 +299,7 @@ def _handle_tool_streaming_completion_error(self, transaction, exc): # Create error event tool_event_dict = _construct_base_tool_event_dict( - strands_attrs, tool_results, transaction, linking_metadata, "strands" + strands_attrs, tool_results, transaction, linking_metadata ) tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmTool", tool_event_dict) @@ -372,7 +389,7 @@ async def __anext__(self): self._nr_on_stop_iteration(self, transaction) raise except Exception as exc: - self._nr_on_error(self, transaction, exc) + self._nr_on_error(self, transaction) raise return return_val @@ -392,7 +409,7 @@ def instrument_agent_agent(module): wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) if hasattr(module.Agent, "invoke_async"): wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) - if hasattr(module.Agent, "_run_loop"): + if hasattr(module.Agent, "stream_async"): wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) From 78b57bb34db0a648c9633e69b0bb2ea7e1bc0495 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:12:06 -0700 Subject: [PATCH 03/31] Add tests file. --- tests/mlmodel_strands/test_agent.py | 372 ++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 tests/mlmodel_strands/test_agent.py diff --git a/tests/mlmodel_strands/test_agent.py b/tests/mlmodel_strands/test_agent.py new file mode 100644 index 0000000000..e2bc83b9dd --- /dev/null +++ b/tests/mlmodel_strands/test_agent.py @@ -0,0 +1,372 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from strands import Agent, tool +from testing_support.fixtures import reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, + tool_events_sans_content, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_error_event_count import validate_transaction_error_event_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import transient_function_wrapper + +tool_recorded_event = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "output": "{'text': 'Hello!'}", + "name": "add_exclamation", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_coro = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_coro", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_agen = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_agen", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +agent_recorded_event = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + +agent_recorded_event_error = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "duration": None, + }, + ) +] + + +# Example tool for testing purposes +@tool +async def add_exclamation(message: str) -> str: + return f"{message}!" + + +@tool +async def throw_exception_coro(message: str) -> str: + raise RuntimeError("Oops") + + +@tool +async def throw_exception_agen(message: str) -> str: + raise RuntimeError("Oops") + yield + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event)) +@validate_custom_events(events_with_context_attrs(agent_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + with WithLlmCustomAttributes({"context": "attr"}): + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = await my_agent.invoke_async('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_stream_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_stream_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = my_agent.stream_async('Add an exclamation to the word "Hello"') + messages = [event["message"]["content"] async for event in response if "message" in event] + + assert len(messages) == 3 + assert messages[0][0]["text"] == "Calling add_exclamation tool" + assert messages[0][1]["toolUse"]["name"] == "add_exclamation" + assert messages[1][0]["toolResult"]["content"][0]["text"] == "Hello!" + assert messages[2][0]["text"] == "Success!" + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(agent_recorded_event) +@validate_custom_events(tool_events_sans_content(tool_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_no_content", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_no_content(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_agent_invoke_disabled_ai_monitoring_events(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event_error) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_error", + scoped_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + rollup_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_error(set_trace_info, single_tool_model): + # Add a wrapper to intentionally force an error in the Agent code + @transient_function_wrapper("strands.agent.agent", "Agent._convert_prompt_to_messages") + def _wrap_convert_prompt_to_messages(wrapped, instance, args, kwargs): + raise ValueError("Oops") + + @_wrap_convert_prompt_to_messages + def _test(): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + my_agent('Add an exclamation to the word "Hello"') # raises ValueError + + with pytest.raises(ValueError): + _test() + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_coro) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_coro_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_coro_runtime_error(set_trace_info, single_tool_model_runtime_error_coro): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_coro, tools=[throw_exception_coro]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_coro"].error_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_agen) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_agen_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_agen_runtime_error(set_trace_info, single_tool_model_runtime_error_agen): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_agen, tools=[throw_exception_agen]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_agen"].error_count == 1 + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_agent_invoke_outside_txn(single_tool_model): + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 From 6c6b40de229b17747d8be650d6798dbe7d4fc43d Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 13:32:24 -0700 Subject: [PATCH 04/31] Cleanup instrumentation. --- newrelic/hooks/mlmodel_strands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 6ac032d8e6..5d17952ff1 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -99,8 +99,7 @@ def wrap_stream_async(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_agent_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata @@ -346,8 +345,7 @@ def wrap_tool_executor__stream(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_tool_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata From cdc2295d545a39f1ee06e4dd960fbcd0f3309ee1 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 14:15:52 -0700 Subject: [PATCH 05/31] Cleanup. Co-authored-by: Tim Pansino --- newrelic/api/time_trace.py | 1 + newrelic/hooks/datastore_elasticsearch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index 7d2ad59a8b..fd0f62fdef 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,6 +361,7 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} + # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 1e87ddddb0..92867d1b83 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,6 +163,7 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) + # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is From c074175aaa3d7bc9de9c8a22353ed99fbb25814a Mon Sep 17 00:00:00 2001 From: umaannamalai <19895951+umaannamalai@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:19:34 +0000 Subject: [PATCH 06/31] [MegaLinter] Apply linters fixes --- newrelic/common/llm_utils.py | 1 - newrelic/hooks/mlmodel_strands.py | 11 ++++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 2ec1136e6d..eebdacfc7f 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -22,4 +22,3 @@ def _get_llm_metadata(transaction): llm_metadata_dict.update(llm_context_attrs) return llm_metadata_dict - diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 5d17952ff1..af954254c1 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -127,9 +127,7 @@ def _record_agent_event_on_stop_iteration(self, transaction): agent_name = strands_attrs.get("agent_name", "agent") agent_id = strands_attrs.get("agent_id", None) - agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata - ) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -234,6 +232,7 @@ def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_ return agent_event_dict + def _handle_agent_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -255,9 +254,7 @@ def _handle_agent_streaming_completion_error(self, transaction): self._nr_ft.__exit__(*sys.exc_info()) # Create error event - agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata - ) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -386,7 +383,7 @@ async def __anext__(self): except StopAsyncIteration: self._nr_on_stop_iteration(self, transaction) raise - except Exception as exc: + except Exception: self._nr_on_error(self, transaction) raise return return_val From 1d9fc03f8ed6fcd34603a971bc59baf82ae5936a Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:16 -0700 Subject: [PATCH 07/31] Strands Mock Model (#1551) * Add strands to tox.ini * Add mock models for strands testing * Add simple test file to validate strands mocking --- tests/mlmodel_strands/_mock_model_provider.py | 99 ++++++++++++ tests/mlmodel_strands/conftest.py | 144 ++++++++++++++++++ tests/mlmodel_strands/test_simple.py | 36 +++++ tox.ini | 12 +- 4 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 tests/mlmodel_strands/_mock_model_provider.py create mode 100644 tests/mlmodel_strands/conftest.py create mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/tests/mlmodel_strands/_mock_model_provider.py b/tests/mlmodel_strands/_mock_model_provider.py new file mode 100644 index 0000000000..e4c9e79930 --- /dev/null +++ b/tests/mlmodel_strands/_mock_model_provider.py @@ -0,0 +1,99 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test setup derived from: https://github.com/strands-agents/sdk-python/blob/main/tests/fixtures/mocked_model_provider.py +# strands Apache 2.0 license: https://github.com/strands-agents/sdk-python/blob/main/LICENSE + +import json +from typing import TypedDict + +from strands.models import Model + + +class RedactionMessage(TypedDict): + redactedUserContent: str + redactedAssistantContent: str + + +class MockedModelProvider(Model): + """A mock implementation of the Model interface for testing purposes. + + This class simulates a model provider by returning pre-defined agent responses + in sequence. It implements the Model interface methods and provides functionality + to stream mock responses as events. + """ + + def __init__(self, agent_responses): + self.agent_responses = agent_responses + self.index = 0 + + def format_chunk(self, event): + return event + + def format_request(self, messages, tool_specs=None, system_prompt=None): + return None + + def get_config(self): + pass + + def update_config(self, **model_config): + pass + + async def structured_output(self, output_model, prompt, system_prompt=None, **kwargs): + pass + + async def stream(self, messages, tool_specs=None, system_prompt=None): + events = self.map_agent_message_to_events(self.agent_responses[self.index]) + for event in events: + yield event + + self.index += 1 + + def map_agent_message_to_events(self, agent_message): + stop_reason = "end_turn" + yield {"messageStart": {"role": "assistant"}} + if agent_message.get("redactedAssistantContent"): + yield {"redactContent": {"redactUserContentMessage": agent_message["redactedUserContent"]}} + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"text": agent_message["redactedAssistantContent"]}}} + yield {"contentBlockStop": {}} + stop_reason = "guardrail_intervened" + else: + for content in agent_message["content"]: + if "reasoningContent" in content: + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"reasoningContent": content["reasoningContent"]}}} + yield {"contentBlockStop": {}} + if "text" in content: + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"text": content["text"]}}} + yield {"contentBlockStop": {}} + if "toolUse" in content: + stop_reason = "tool_use" + yield { + "contentBlockStart": { + "start": { + "toolUse": { + "name": content["toolUse"]["name"], + "toolUseId": content["toolUse"]["toolUseId"], + } + } + } + } + yield { + "contentBlockDelta": {"delta": {"toolUse": {"input": json.dumps(content["toolUse"]["input"])}}} + } + yield {"contentBlockStop": {}} + + yield {"messageStop": {"stopReason": stop_reason}} diff --git a/tests/mlmodel_strands/conftest.py b/tests/mlmodel_strands/conftest.py new file mode 100644 index 0000000000..b810161f6a --- /dev/null +++ b/tests/mlmodel_strands/conftest.py @@ -0,0 +1,144 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from _mock_model_provider import MockedModelProvider +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture +from testing_support.ml_testing_utils import set_trace_info + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slowdowns. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ai_monitoring.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_strands)", default_settings=_default_settings +) + + +@pytest.fixture +def single_tool_model(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def single_tool_model_error(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + # Set arguments to an invalid type to trigger error in tool + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": 12}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def multi_tool_model(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 5, "b": 3}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Goodbye"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 123, "b": 2}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def multi_tool_model_error(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 5, "b": 3}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Goodbye"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + # Set insufficient arguments to trigger error in tool + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 123}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py new file mode 100644 index 0000000000..ae24003fab --- /dev/null +++ b/tests/mlmodel_strands/test_simple.py @@ -0,0 +1,36 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from strands import Agent, tool + +from newrelic.api.background_task import background_task + + +# Example tool for testing purposes +@tool +def add_exclamation(message: str) -> str: + return f"{message}!" + + +# TODO: Remove this file once all real tests are in place + + +@background_task() +def test_simple_run_agent(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent("Run the tools.") + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 diff --git a/tox.ini b/tox.ini index 39148b657f..ace7839db3 100644 --- a/tox.ini +++ b/tox.ini @@ -182,6 +182,7 @@ envlist = python-logger_structlog-{py38,py39,py310,py311,py312,py313,py314,pypy311}-structloglatest, python-mlmodel_autogen-{py310,py311,py312,py313,py314,pypy311}-autogen061, python-mlmodel_autogen-{py310,py311,py312,py313,py314,pypy311}-autogenlatest, + python-mlmodel_strands-{py310,py311,py312,py313}-strandslatest, python-mlmodel_gemini-{py39,py310,py311,py312,py313,py314}, python-mlmodel_langchain-{py39,py310,py311,py312,py313}, ;; Package not ready for Python 3.14 (type annotations not updated) @@ -440,6 +441,8 @@ deps = mlmodel_langchain: faiss-cpu mlmodel_langchain: mock mlmodel_langchain: asyncio + mlmodel_strands: strands-agents[openai] + mlmodel_strands: strands-agents-tools logger_loguru-logurulatest: loguru logger_structlog-structloglatest: structlog messagebroker_pika-pikalatest: pika @@ -510,6 +513,7 @@ changedir = application_celery: tests/application_celery component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest + component_graphenedjango: tests/component_graphenedjango component_graphqlserver: tests/component_graphqlserver component_tastypie: tests/component_tastypie coroutines_asyncio: tests/coroutines_asyncio @@ -521,17 +525,17 @@ changedir = datastore_cassandradriver: tests/datastore_cassandradriver datastore_elasticsearch: tests/datastore_elasticsearch datastore_firestore: tests/datastore_firestore - datastore_oracledb: tests/datastore_oracledb datastore_memcache: tests/datastore_memcache + datastore_motor: tests/datastore_motor datastore_mysql: tests/datastore_mysql datastore_mysqldb: tests/datastore_mysqldb + datastore_oracledb: tests/datastore_oracledb datastore_postgresql: tests/datastore_postgresql datastore_psycopg: tests/datastore_psycopg datastore_psycopg2: tests/datastore_psycopg2 datastore_psycopg2cffi: tests/datastore_psycopg2cffi datastore_pylibmc: tests/datastore_pylibmc datastore_pymemcache: tests/datastore_pymemcache - datastore_motor: tests/datastore_motor datastore_pymongo: tests/datastore_pymongo datastore_pymssql: tests/datastore_pymssql datastore_pymysql: tests/datastore_pymysql @@ -539,8 +543,8 @@ changedir = datastore_pysolr: tests/datastore_pysolr datastore_redis: tests/datastore_redis datastore_rediscluster: tests/datastore_rediscluster - datastore_valkey: tests/datastore_valkey datastore_sqlite: tests/datastore_sqlite + datastore_valkey: tests/datastore_valkey external_aiobotocore: tests/external_aiobotocore external_botocore: tests/external_botocore external_feedparser: tests/external_feedparser @@ -561,7 +565,6 @@ changedir = framework_fastapi: tests/framework_fastapi framework_flask: tests/framework_flask framework_graphene: tests/framework_graphene - component_graphenedjango: tests/component_graphenedjango framework_graphql: tests/framework_graphql framework_grpc: tests/framework_grpc framework_pyramid: tests/framework_pyramid @@ -581,6 +584,7 @@ changedir = mlmodel_langchain: tests/mlmodel_langchain mlmodel_openai: tests/mlmodel_openai mlmodel_sklearn: tests/mlmodel_sklearn + mlmodel_strands: tests/mlmodel_strands template_genshi: tests/template_genshi template_jinja2: tests/template_jinja2 template_mako: tests/template_mako From e1cbba0c88fccff5994f0c81102d1c62cb633c97 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 27 Oct 2025 12:44:35 -0700 Subject: [PATCH 08/31] Add baseline instrumentation. --- newrelic/api/error_trace.py | 23 +- newrelic/api/time_trace.py | 1 - newrelic/common/llm_utils.py | 46 ++ newrelic/config.py | 7 + newrelic/hooks/datastore_elasticsearch.py | 1 - newrelic/hooks/mlmodel_strands.py | 408 ++++++++++++++++++ tests/mlmodel_strands/conftest.py | 25 +- tests/mlmodel_strands/test_simple.py | 36 -- tests/testing_support/fixtures.py | 2 +- .../validators/validate_custom_event.py | 4 +- .../validate_error_event_collector_json.py | 2 +- .../validate_transaction_error_event_count.py | 4 +- 12 files changed, 507 insertions(+), 52 deletions(-) create mode 100644 newrelic/common/llm_utils.py create mode 100644 newrelic/hooks/mlmodel_strands.py delete mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index db63c54316..c94ed34dc9 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -15,6 +15,7 @@ import functools from newrelic.api.time_trace import current_trace, notice_error +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object @@ -43,17 +44,25 @@ def __exit__(self, exc, value, tb): ) -def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None): - def wrapper(wrapped, instance, args, kwargs): - parent = current_trace() +def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None, async_wrapper=None): + def literal_wrapper(wrapped, instance, args, kwargs): + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + if not wrapper: + parent = current_trace() + if not parent: + return wrapped(*args, **kwargs) + else: + parent = None - if parent is None: - return wrapped(*args, **kwargs) + trace = ErrorTrace(ignore, expected, status_code, parent=parent) + + if wrapper: + return wrapper(wrapped, trace)(*args, **kwargs) - with ErrorTrace(ignore, expected, status_code, parent=parent): + with trace: return wrapped(*args, **kwargs) - return FunctionWrapper(wrapped, wrapper) + return FunctionWrapper(wrapped, literal_wrapper) def error_trace(ignore=None, expected=None, status_code=None): diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index fd0f62fdef..7d2ad59a8b 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,7 +361,6 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} - # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py new file mode 100644 index 0000000000..0ae8575477 --- /dev/null +++ b/newrelic/common/llm_utils.py @@ -0,0 +1,46 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +_logger = logging.getLogger(__name__) +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" + + +def _get_llm_metadata(transaction): + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": vendor_name, + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 21ce996f6c..806563a1f6 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2946,6 +2946,13 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition("strands.agent.agent", "newrelic.hooks.mlmodel_strands", "instrument_agent_agent") + _process_module_definition( + "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" + ) + + _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") + _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") _process_module_definition( "mcp.server.fastmcp.tools.tool_manager", diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 92867d1b83..1e87ddddb0 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,7 +163,6 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) - # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py new file mode 100644 index 0000000000..8bb977808f --- /dev/null +++ b/newrelic/hooks/mlmodel_strands.py @@ -0,0 +1,408 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid + +from newrelic.api.error_trace import ErrorTraceWrapper +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.core.context import ContextOf + +_logger = logging.getLogger(__name__) +STRANDS_VERSION = get_package_version("strands-agents") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." +TOOL_OUTPUT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record output of tool call. Please report this issue to New Relic Support." + + +def wrap_agent__call__(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + # Make a copy of the invocation state before we mutate it + if "invocation_state" in bound_args: + invocation_state = bound_args["invocation_state"] = dict(bound_args["invocation_state"] or {}) + + # Attempt to save the current transaction context into the invocation state dictionary + invocation_state["_nr_transaction"] = trace + except Exception: + return wrapped(*args, **kwargs) + else: + return wrapped(**bound_args) + + +async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): + # If there's already a transaction, don't propagate anything here + if current_transaction(): + return await wrapped(*args, **kwargs) + + try: + # Grab the trace context we should be running under and pass it to ContextOf + bound_args = bind_args(wrapped, args, kwargs) + invocation_state = bound_args["invocation_state"] or {} + trace = invocation_state.pop("_nr_transaction", None) + except Exception: + return await wrapped(*args, **kwargs) + + # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + if trace: + with ContextOf(trace=trace): + return await wrapped(*args, **kwargs) + else: + return await wrapped(*args, **kwargs) + + +def wrap_stream_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + func_name = callable_name(wrapped) + agent_name = getattr(instance, "name", "agent") + function_trace_name = f"{func_name}/{agent_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + agent_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_agent_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} + return proxied_return_val + + +def _record_agent_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _record_tool_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, linking_metadata): + try: + try: + tool_output = tool_results[-1]["content"][0] if tool_results else None + error = tool_results[-1]["status"] == "error" + except Exception: + tool_output = None + error = False + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_name = strands_attrs.get("tool_name", "tool") + tool_id = strands_attrs.get("tool_id", None) + run_id = strands_attrs.get("run_id", None) + tool_input = strands_attrs.get("tool_input", None) + agent_name = strands_attrs.get("agent_name", "agent") + settings = transaction.settings or global_settings() + + tool_event_dict = { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "strands", + "ingest_source": "Python", + } + # Set error flag if the status shows an error was caught, + # it will be reported further down in the instrumentation. + if error: + tool_event_dict["error"] = True + + if settings.ai_monitoring.record_content.enabled: + tool_event_dict.update({"input": tool_input}) + tool_event_dict.update({"output": tool_output}) + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def _handle_agent_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"agent_id": agent_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _handle_tool_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + tool_id = strands_attrs["tool_id"] + + # We expect this to never have any output since this is an error case, + # but if it does we will report it. + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"tool_id": tool_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata, "strands" + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def wrap_tool_executor__stream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + # Grab tool data + bound_args = bind_args(wrapped, args, kwargs) + agent_name = getattr(bound_args.get("agent"), "name", "agent") + tool_use = bound_args.get("tool_use", {}) + + run_id = tool_use.get("toolUseId", "") + tool_name = tool_use.get("name", "tool") + _input = tool_use.get("input") + tool_input = str(_input) if _input else None + tool_results = bound_args.get("tool_results", []) + + func_name = callable_name(wrapped) + function_trace_name = f"{func_name}/{tool_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/tool/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + tool_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_tool_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = { + "tool_results": tool_results, + "tool_name": tool_name, + "tool_id": tool_id, + "run_id": run_id, + "tool_input": tool_input, + "agent_name": agent_name, + } + return proxied_return_val + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped, on_stop_iteration, on_error): + super().__init__(wrapped) + self._nr_on_stop_iteration = on_stop_iteration + self._nr_on_error = on_error + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + except StopAsyncIteration: + self._nr_on_stop_iteration(self, transaction) + raise + except Exception as exc: + self._nr_on_error(self, transaction, exc) + raise + return return_val + + async def aclose(self): + return await super().aclose() + + +def wrap_ToolRegister_register_tool(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + bound_args["tool"]._tool_func = ErrorTraceWrapper(bound_args["tool"]._tool_func) + return wrapped(*args, **kwargs) + + +def instrument_agent_agent(module): + if hasattr(module, "Agent"): + if hasattr(module.Agent, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) + if hasattr(module.Agent, "invoke_async"): + wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) + if hasattr(module.Agent, "_run_loop"): + wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) + + +def instrument_tools_executors__executor(module): + if hasattr(module, "ToolExecutor"): + if hasattr(module.ToolExecutor, "_stream"): + wrap_function_wrapper(module, "ToolExecutor._stream", wrap_tool_executor__stream) + + +def instrument_tools_registry(module): + if hasattr(module, "ToolRegistry"): + if hasattr(module.ToolRegistry, "register_tool"): + wrap_function_wrapper(module, "ToolRegistry.register_tool", wrap_ToolRegister_register_tool) diff --git a/tests/mlmodel_strands/conftest.py b/tests/mlmodel_strands/conftest.py index b810161f6a..a2ad9b8dd0 100644 --- a/tests/mlmodel_strands/conftest.py +++ b/tests/mlmodel_strands/conftest.py @@ -14,6 +14,7 @@ import pytest from _mock_model_provider import MockedModelProvider +from testing_support.fixture.event_loop import event_loop as loop from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture from testing_support.ml_testing_utils import set_trace_info @@ -50,15 +51,33 @@ def single_tool_model(): @pytest.fixture -def single_tool_model_error(): +def single_tool_model_runtime_error_coro(): model = MockedModelProvider( [ { "role": "assistant", "content": [ - {"text": "Calling add_exclamation tool"}, + {"text": "Calling throw_exception_coro tool"}, + # Set arguments to an invalid type to trigger error in tool + {"toolUse": {"name": "throw_exception_coro", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def single_tool_model_runtime_error_agen(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling throw_exception_agen tool"}, # Set arguments to an invalid type to trigger error in tool - {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": 12}}}, + {"toolUse": {"name": "throw_exception_agen", "toolUseId": "123", "input": {"message": "Hello"}}}, ], }, {"role": "assistant", "content": [{"text": "Success!"}]}, diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py deleted file mode 100644 index ae24003fab..0000000000 --- a/tests/mlmodel_strands/test_simple.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from strands import Agent, tool - -from newrelic.api.background_task import background_task - - -# Example tool for testing purposes -@tool -def add_exclamation(message: str) -> str: - return f"{message}!" - - -# TODO: Remove this file once all real tests are in place - - -@background_task() -def test_simple_run_agent(set_trace_info, single_tool_model): - set_trace_info() - my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) - - response = my_agent("Run the tools.") - assert response.message["content"][0]["text"] == "Success!" - assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 3d93e06e30..540e44f70c 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -797,7 +797,7 @@ def _bind_params(transaction, *args, **kwargs): transaction = _bind_params(*args, **kwargs) error_events = transaction.error_events(instance.stats_table) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for sample in error_events: assert isinstance(sample, list) assert len(sample) == 3 diff --git a/tests/testing_support/validators/validate_custom_event.py b/tests/testing_support/validators/validate_custom_event.py index deeef7fb25..c3cf78032a 100644 --- a/tests/testing_support/validators/validate_custom_event.py +++ b/tests/testing_support/validators/validate_custom_event.py @@ -61,7 +61,9 @@ def _validate_custom_event_count(wrapped, instance, args, kwargs): raise else: stats = core_application_stats_engine(None) - assert stats.custom_events.num_samples == count + assert stats.custom_events.num_samples == count, ( + f"Expected: {count}, Got: {stats.custom_events.num_samples}. Events: {list(stats.custom_events)}" + ) return result diff --git a/tests/testing_support/validators/validate_error_event_collector_json.py b/tests/testing_support/validators/validate_error_event_collector_json.py index d1cec3a558..27ea76f3a3 100644 --- a/tests/testing_support/validators/validate_error_event_collector_json.py +++ b/tests/testing_support/validators/validate_error_event_collector_json.py @@ -52,7 +52,7 @@ def _validate_error_event_collector_json(wrapped, instance, args, kwargs): error_events = decoded_json[2] - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for event in error_events: # event is an array containing intrinsics, user-attributes, # and agent-attributes diff --git a/tests/testing_support/validators/validate_transaction_error_event_count.py b/tests/testing_support/validators/validate_transaction_error_event_count.py index b41a52330f..f5e8c0b206 100644 --- a/tests/testing_support/validators/validate_transaction_error_event_count.py +++ b/tests/testing_support/validators/validate_transaction_error_event_count.py @@ -28,7 +28,9 @@ def _validate_error_event_on_stats_engine(wrapped, instance, args, kwargs): raise else: error_events = list(instance.error_events) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, ( + f"Expected: {num_errors}, Got: {len(error_events)}. Errors: {error_events}" + ) return result From 5cf6c37cb1e71ab15182463ceb6b443dc9d8ffad Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:11:49 -0700 Subject: [PATCH 09/31] Add tool and agent instrumentation. --- newrelic/common/llm_utils.py | 21 ------------------- newrelic/config.py | 1 - newrelic/hooks/mlmodel_strands.py | 35 +++++++++++++++++++++++-------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 0ae8575477..2ec1136e6d 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - -_logger = logging.getLogger(__name__) -RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" - def _get_llm_metadata(transaction): # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events @@ -28,19 +23,3 @@ def _get_llm_metadata(transaction): return llm_metadata_dict - -def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): - try: - agent_event_dict = { - "id": agent_id, - "name": agent_name, - "span_id": linking_metadata.get("span.id"), - "trace_id": linking_metadata.get("trace.id"), - "vendor": vendor_name, - "ingest_source": "Python", - } - agent_event_dict.update(_get_llm_metadata(transaction)) - except Exception: - agent_event_dict = {} - - return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 806563a1f6..09a82e7c9a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2950,7 +2950,6 @@ def _process_module_builtin_defaults(): _process_module_definition( "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" ) - _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 8bb977808f..6ac032d8e6 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -20,7 +20,7 @@ from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace, get_trace_linking_metadata from newrelic.api.transaction import current_transaction -from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.llm_utils import _get_llm_metadata from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version @@ -67,7 +67,7 @@ async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): except Exception: return await wrapped(*args, **kwargs) - # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + # If we find a transaction to propagate, use it. Otherwise, just call wrapped. if trace: with ContextOf(trace=trace): return await wrapped(*args, **kwargs) @@ -129,7 +129,7 @@ def _record_agent_event_on_stop_iteration(self, transaction): agent_name = strands_attrs.get("agent_name", "agent") agent_id = strands_attrs.get("agent_id", None) agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -209,6 +209,7 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li if settings.ai_monitoring.record_content.enabled: tool_event_dict.update({"input": tool_input}) + # In error cases, the output will hold the error message tool_event_dict.update({"output": tool_output}) tool_event_dict.update(_get_llm_metadata(transaction)) except Exception: @@ -218,7 +219,23 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li return tool_event_dict -def _handle_agent_streaming_completion_error(self, transaction, exc): +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "strands", + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict + +def _handle_agent_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -240,7 +257,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): # Create error event agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -253,7 +270,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): self._nr_strands_attrs.clear() -def _handle_tool_streaming_completion_error(self, transaction, exc): +def _handle_tool_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -282,7 +299,7 @@ def _handle_tool_streaming_completion_error(self, transaction, exc): # Create error event tool_event_dict = _construct_base_tool_event_dict( - strands_attrs, tool_results, transaction, linking_metadata, "strands" + strands_attrs, tool_results, transaction, linking_metadata ) tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmTool", tool_event_dict) @@ -372,7 +389,7 @@ async def __anext__(self): self._nr_on_stop_iteration(self, transaction) raise except Exception as exc: - self._nr_on_error(self, transaction, exc) + self._nr_on_error(self, transaction) raise return return_val @@ -392,7 +409,7 @@ def instrument_agent_agent(module): wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) if hasattr(module.Agent, "invoke_async"): wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) - if hasattr(module.Agent, "_run_loop"): + if hasattr(module.Agent, "stream_async"): wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) From 1dfaf5592162a57a23c8b6cbec55a0f178e96a48 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:12:06 -0700 Subject: [PATCH 10/31] Add tests file. --- tests/mlmodel_strands/test_agent.py | 372 ++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 tests/mlmodel_strands/test_agent.py diff --git a/tests/mlmodel_strands/test_agent.py b/tests/mlmodel_strands/test_agent.py new file mode 100644 index 0000000000..e2bc83b9dd --- /dev/null +++ b/tests/mlmodel_strands/test_agent.py @@ -0,0 +1,372 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from strands import Agent, tool +from testing_support.fixtures import reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, + tool_events_sans_content, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_error_event_count import validate_transaction_error_event_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import transient_function_wrapper + +tool_recorded_event = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "output": "{'text': 'Hello!'}", + "name": "add_exclamation", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_coro = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_coro", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_agen = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_agen", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +agent_recorded_event = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + +agent_recorded_event_error = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "duration": None, + }, + ) +] + + +# Example tool for testing purposes +@tool +async def add_exclamation(message: str) -> str: + return f"{message}!" + + +@tool +async def throw_exception_coro(message: str) -> str: + raise RuntimeError("Oops") + + +@tool +async def throw_exception_agen(message: str) -> str: + raise RuntimeError("Oops") + yield + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event)) +@validate_custom_events(events_with_context_attrs(agent_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + with WithLlmCustomAttributes({"context": "attr"}): + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = await my_agent.invoke_async('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_stream_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_stream_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = my_agent.stream_async('Add an exclamation to the word "Hello"') + messages = [event["message"]["content"] async for event in response if "message" in event] + + assert len(messages) == 3 + assert messages[0][0]["text"] == "Calling add_exclamation tool" + assert messages[0][1]["toolUse"]["name"] == "add_exclamation" + assert messages[1][0]["toolResult"]["content"][0]["text"] == "Hello!" + assert messages[2][0]["text"] == "Success!" + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(agent_recorded_event) +@validate_custom_events(tool_events_sans_content(tool_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_no_content", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_no_content(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_agent_invoke_disabled_ai_monitoring_events(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event_error) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_error", + scoped_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + rollup_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_error(set_trace_info, single_tool_model): + # Add a wrapper to intentionally force an error in the Agent code + @transient_function_wrapper("strands.agent.agent", "Agent._convert_prompt_to_messages") + def _wrap_convert_prompt_to_messages(wrapped, instance, args, kwargs): + raise ValueError("Oops") + + @_wrap_convert_prompt_to_messages + def _test(): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + my_agent('Add an exclamation to the word "Hello"') # raises ValueError + + with pytest.raises(ValueError): + _test() + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_coro) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_coro_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_coro_runtime_error(set_trace_info, single_tool_model_runtime_error_coro): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_coro, tools=[throw_exception_coro]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_coro"].error_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_agen) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_agen_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_agen_runtime_error(set_trace_info, single_tool_model_runtime_error_agen): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_agen, tools=[throw_exception_agen]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_agen"].error_count == 1 + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_agent_invoke_outside_txn(single_tool_model): + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 From df369a736e3fa9984a16d129809908b065bb599d Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 13:32:24 -0700 Subject: [PATCH 11/31] Cleanup instrumentation. --- newrelic/hooks/mlmodel_strands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 6ac032d8e6..5d17952ff1 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -99,8 +99,7 @@ def wrap_stream_async(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_agent_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata @@ -346,8 +345,7 @@ def wrap_tool_executor__stream(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_tool_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata From 69e1c88f1f961b03de58b9bfc8e4a98fe9692f53 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 14:15:52 -0700 Subject: [PATCH 12/31] Cleanup. Co-authored-by: Tim Pansino --- newrelic/api/time_trace.py | 1 + newrelic/hooks/datastore_elasticsearch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index 7d2ad59a8b..fd0f62fdef 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,6 +361,7 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} + # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 1e87ddddb0..92867d1b83 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,6 +163,7 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) + # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is From a5a313338838bcf7f363aa9c1fc7925e3b12b0ae Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 15:07:56 -0700 Subject: [PATCH 13/31] Handle additional args in mock model. --- tests/mlmodel_strands/_mock_model_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mlmodel_strands/_mock_model_provider.py b/tests/mlmodel_strands/_mock_model_provider.py index e4c9e79930..ef60e13bad 100644 --- a/tests/mlmodel_strands/_mock_model_provider.py +++ b/tests/mlmodel_strands/_mock_model_provider.py @@ -41,7 +41,7 @@ def __init__(self, agent_responses): def format_chunk(self, event): return event - def format_request(self, messages, tool_specs=None, system_prompt=None): + def format_request(self, messages, tool_specs=None, system_prompt=None, **kwargs): return None def get_config(self): @@ -53,7 +53,7 @@ def update_config(self, **model_config): async def structured_output(self, output_model, prompt, system_prompt=None, **kwargs): pass - async def stream(self, messages, tool_specs=None, system_prompt=None): + async def stream(self, messages, tool_specs=None, system_prompt=None, **kwargs): events = self.map_agent_message_to_events(self.agent_responses[self.index]) for event in events: yield event From e2ad09dd63100fa595d6d32928fd28fb2dc761ee Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 3 Nov 2025 19:12:11 -0800 Subject: [PATCH 14/31] Add test to force exception and exercise _handle_tool_streaming_completion_error. --- newrelic/hooks/mlmodel_strands.py | 4 +++ tests/mlmodel_strands/test_agent.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index af954254c1..d09129b80d 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -298,6 +298,10 @@ def _handle_tool_streaming_completion_error(self, transaction): strands_attrs, tool_results, transaction, linking_metadata ) tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + # Ensure error flag is set to True in case the tool_results did not indicate an error + if "error" not in tool_event_dict: + tool_event_dict.update({"error": True}) + transaction.record_custom_event("LlmTool", tool_event_dict) except Exception: diff --git a/tests/mlmodel_strands/test_agent.py b/tests/mlmodel_strands/test_agent.py index e2bc83b9dd..af685668ad 100644 --- a/tests/mlmodel_strands/test_agent.py +++ b/tests/mlmodel_strands/test_agent.py @@ -51,6 +51,24 @@ ) ] +tool_recorded_event_forced_internal_error = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "add_exclamation", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + "error": True, + }, + ) +] tool_recorded_event_error_coro = [ ( @@ -362,6 +380,43 @@ def test_agent_invoke_tool_agen_runtime_error(set_trace_info, single_tool_model_ assert response.metrics.tool_metrics["throw_exception_agen"].error_count == 1 +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event) +@validate_custom_events(tool_recorded_event_forced_internal_error) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_tool_forced_exception", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_tool_forced_exception(set_trace_info, single_tool_model): + # Add a wrapper to intentionally force an error in the ToolExecutor._stream code to hit the exception path in + # the AsyncGeneratorProxy + @transient_function_wrapper("strands.hooks.events", "BeforeToolCallEvent.__init__") + def _wrap_BeforeToolCallEvent_init(wrapped, instance, args, kwargs): + raise ValueError("Oops") + + @_wrap_BeforeToolCallEvent_init + def _test(): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + my_agent('Add an exclamation to the word "Hello"') + + # This will not explicitly raise a ValueError when running the test but we are still able to capture it in the error trace + _test() + + @reset_core_stats_engine() @validate_custom_event_count(count=0) def test_agent_invoke_outside_txn(single_tool_model): From 4213bd3f5c09f76499c7ee5cb74d0e8beed73b8b Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:16 -0700 Subject: [PATCH 15/31] Strands Mock Model (#1551) * Add strands to tox.ini * Add mock models for strands testing * Add simple test file to validate strands mocking --- tests/mlmodel_strands/_mock_model_provider.py | 99 ++++++++++++ tests/mlmodel_strands/conftest.py | 144 ++++++++++++++++++ tests/mlmodel_strands/test_simple.py | 36 +++++ tox.ini | 12 +- 4 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 tests/mlmodel_strands/_mock_model_provider.py create mode 100644 tests/mlmodel_strands/conftest.py create mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/tests/mlmodel_strands/_mock_model_provider.py b/tests/mlmodel_strands/_mock_model_provider.py new file mode 100644 index 0000000000..e4c9e79930 --- /dev/null +++ b/tests/mlmodel_strands/_mock_model_provider.py @@ -0,0 +1,99 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test setup derived from: https://github.com/strands-agents/sdk-python/blob/main/tests/fixtures/mocked_model_provider.py +# strands Apache 2.0 license: https://github.com/strands-agents/sdk-python/blob/main/LICENSE + +import json +from typing import TypedDict + +from strands.models import Model + + +class RedactionMessage(TypedDict): + redactedUserContent: str + redactedAssistantContent: str + + +class MockedModelProvider(Model): + """A mock implementation of the Model interface for testing purposes. + + This class simulates a model provider by returning pre-defined agent responses + in sequence. It implements the Model interface methods and provides functionality + to stream mock responses as events. + """ + + def __init__(self, agent_responses): + self.agent_responses = agent_responses + self.index = 0 + + def format_chunk(self, event): + return event + + def format_request(self, messages, tool_specs=None, system_prompt=None): + return None + + def get_config(self): + pass + + def update_config(self, **model_config): + pass + + async def structured_output(self, output_model, prompt, system_prompt=None, **kwargs): + pass + + async def stream(self, messages, tool_specs=None, system_prompt=None): + events = self.map_agent_message_to_events(self.agent_responses[self.index]) + for event in events: + yield event + + self.index += 1 + + def map_agent_message_to_events(self, agent_message): + stop_reason = "end_turn" + yield {"messageStart": {"role": "assistant"}} + if agent_message.get("redactedAssistantContent"): + yield {"redactContent": {"redactUserContentMessage": agent_message["redactedUserContent"]}} + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"text": agent_message["redactedAssistantContent"]}}} + yield {"contentBlockStop": {}} + stop_reason = "guardrail_intervened" + else: + for content in agent_message["content"]: + if "reasoningContent" in content: + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"reasoningContent": content["reasoningContent"]}}} + yield {"contentBlockStop": {}} + if "text" in content: + yield {"contentBlockStart": {"start": {}}} + yield {"contentBlockDelta": {"delta": {"text": content["text"]}}} + yield {"contentBlockStop": {}} + if "toolUse" in content: + stop_reason = "tool_use" + yield { + "contentBlockStart": { + "start": { + "toolUse": { + "name": content["toolUse"]["name"], + "toolUseId": content["toolUse"]["toolUseId"], + } + } + } + } + yield { + "contentBlockDelta": {"delta": {"toolUse": {"input": json.dumps(content["toolUse"]["input"])}}} + } + yield {"contentBlockStop": {}} + + yield {"messageStop": {"stopReason": stop_reason}} diff --git a/tests/mlmodel_strands/conftest.py b/tests/mlmodel_strands/conftest.py new file mode 100644 index 0000000000..b810161f6a --- /dev/null +++ b/tests/mlmodel_strands/conftest.py @@ -0,0 +1,144 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from _mock_model_provider import MockedModelProvider +from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture +from testing_support.ml_testing_utils import set_trace_info + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slowdowns. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "ai_monitoring.enabled": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (mlmodel_strands)", default_settings=_default_settings +) + + +@pytest.fixture +def single_tool_model(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def single_tool_model_error(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + # Set arguments to an invalid type to trigger error in tool + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": 12}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def multi_tool_model(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 5, "b": 3}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Goodbye"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 123, "b": 2}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def multi_tool_model_error(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 5, "b": 3}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling add_exclamation tool"}, + {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": "Goodbye"}}}, + ], + }, + { + "role": "assistant", + "content": [ + {"text": "Calling compute_sum tool"}, + # Set insufficient arguments to trigger error in tool + {"toolUse": {"name": "compute_sum", "toolUseId": "123", "input": {"a": 123}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py new file mode 100644 index 0000000000..ae24003fab --- /dev/null +++ b/tests/mlmodel_strands/test_simple.py @@ -0,0 +1,36 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from strands import Agent, tool + +from newrelic.api.background_task import background_task + + +# Example tool for testing purposes +@tool +def add_exclamation(message: str) -> str: + return f"{message}!" + + +# TODO: Remove this file once all real tests are in place + + +@background_task() +def test_simple_run_agent(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent("Run the tools.") + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 diff --git a/tox.ini b/tox.ini index 39148b657f..ace7839db3 100644 --- a/tox.ini +++ b/tox.ini @@ -182,6 +182,7 @@ envlist = python-logger_structlog-{py38,py39,py310,py311,py312,py313,py314,pypy311}-structloglatest, python-mlmodel_autogen-{py310,py311,py312,py313,py314,pypy311}-autogen061, python-mlmodel_autogen-{py310,py311,py312,py313,py314,pypy311}-autogenlatest, + python-mlmodel_strands-{py310,py311,py312,py313}-strandslatest, python-mlmodel_gemini-{py39,py310,py311,py312,py313,py314}, python-mlmodel_langchain-{py39,py310,py311,py312,py313}, ;; Package not ready for Python 3.14 (type annotations not updated) @@ -440,6 +441,8 @@ deps = mlmodel_langchain: faiss-cpu mlmodel_langchain: mock mlmodel_langchain: asyncio + mlmodel_strands: strands-agents[openai] + mlmodel_strands: strands-agents-tools logger_loguru-logurulatest: loguru logger_structlog-structloglatest: structlog messagebroker_pika-pikalatest: pika @@ -510,6 +513,7 @@ changedir = application_celery: tests/application_celery component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest + component_graphenedjango: tests/component_graphenedjango component_graphqlserver: tests/component_graphqlserver component_tastypie: tests/component_tastypie coroutines_asyncio: tests/coroutines_asyncio @@ -521,17 +525,17 @@ changedir = datastore_cassandradriver: tests/datastore_cassandradriver datastore_elasticsearch: tests/datastore_elasticsearch datastore_firestore: tests/datastore_firestore - datastore_oracledb: tests/datastore_oracledb datastore_memcache: tests/datastore_memcache + datastore_motor: tests/datastore_motor datastore_mysql: tests/datastore_mysql datastore_mysqldb: tests/datastore_mysqldb + datastore_oracledb: tests/datastore_oracledb datastore_postgresql: tests/datastore_postgresql datastore_psycopg: tests/datastore_psycopg datastore_psycopg2: tests/datastore_psycopg2 datastore_psycopg2cffi: tests/datastore_psycopg2cffi datastore_pylibmc: tests/datastore_pylibmc datastore_pymemcache: tests/datastore_pymemcache - datastore_motor: tests/datastore_motor datastore_pymongo: tests/datastore_pymongo datastore_pymssql: tests/datastore_pymssql datastore_pymysql: tests/datastore_pymysql @@ -539,8 +543,8 @@ changedir = datastore_pysolr: tests/datastore_pysolr datastore_redis: tests/datastore_redis datastore_rediscluster: tests/datastore_rediscluster - datastore_valkey: tests/datastore_valkey datastore_sqlite: tests/datastore_sqlite + datastore_valkey: tests/datastore_valkey external_aiobotocore: tests/external_aiobotocore external_botocore: tests/external_botocore external_feedparser: tests/external_feedparser @@ -561,7 +565,6 @@ changedir = framework_fastapi: tests/framework_fastapi framework_flask: tests/framework_flask framework_graphene: tests/framework_graphene - component_graphenedjango: tests/component_graphenedjango framework_graphql: tests/framework_graphql framework_grpc: tests/framework_grpc framework_pyramid: tests/framework_pyramid @@ -581,6 +584,7 @@ changedir = mlmodel_langchain: tests/mlmodel_langchain mlmodel_openai: tests/mlmodel_openai mlmodel_sklearn: tests/mlmodel_sklearn + mlmodel_strands: tests/mlmodel_strands template_genshi: tests/template_genshi template_jinja2: tests/template_jinja2 template_mako: tests/template_mako From 520bd9ac8ade66065c05a6345440b0e52ae430e7 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 27 Oct 2025 12:44:35 -0700 Subject: [PATCH 16/31] Add baseline instrumentation. --- newrelic/api/error_trace.py | 23 +- newrelic/api/time_trace.py | 1 - newrelic/common/llm_utils.py | 46 ++ newrelic/config.py | 7 + newrelic/hooks/datastore_elasticsearch.py | 1 - newrelic/hooks/mlmodel_strands.py | 408 ++++++++++++++++++ tests/mlmodel_strands/conftest.py | 25 +- tests/mlmodel_strands/test_simple.py | 36 -- tests/testing_support/fixtures.py | 2 +- .../validators/validate_custom_event.py | 4 +- .../validate_error_event_collector_json.py | 2 +- .../validate_transaction_error_event_count.py | 4 +- 12 files changed, 507 insertions(+), 52 deletions(-) create mode 100644 newrelic/common/llm_utils.py create mode 100644 newrelic/hooks/mlmodel_strands.py delete mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index db63c54316..c94ed34dc9 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -15,6 +15,7 @@ import functools from newrelic.api.time_trace import current_trace, notice_error +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object @@ -43,17 +44,25 @@ def __exit__(self, exc, value, tb): ) -def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None): - def wrapper(wrapped, instance, args, kwargs): - parent = current_trace() +def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None, async_wrapper=None): + def literal_wrapper(wrapped, instance, args, kwargs): + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + if not wrapper: + parent = current_trace() + if not parent: + return wrapped(*args, **kwargs) + else: + parent = None - if parent is None: - return wrapped(*args, **kwargs) + trace = ErrorTrace(ignore, expected, status_code, parent=parent) + + if wrapper: + return wrapper(wrapped, trace)(*args, **kwargs) - with ErrorTrace(ignore, expected, status_code, parent=parent): + with trace: return wrapped(*args, **kwargs) - return FunctionWrapper(wrapped, wrapper) + return FunctionWrapper(wrapped, literal_wrapper) def error_trace(ignore=None, expected=None, status_code=None): diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index fd0f62fdef..7d2ad59a8b 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,7 +361,6 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} - # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py new file mode 100644 index 0000000000..0ae8575477 --- /dev/null +++ b/newrelic/common/llm_utils.py @@ -0,0 +1,46 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +_logger = logging.getLogger(__name__) +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" + + +def _get_llm_metadata(transaction): + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": vendor_name, + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 21ce996f6c..806563a1f6 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2946,6 +2946,13 @@ def _process_module_builtin_defaults(): "newrelic.hooks.mlmodel_autogen", "instrument_autogen_agentchat_agents__assistant_agent", ) + _process_module_definition("strands.agent.agent", "newrelic.hooks.mlmodel_strands", "instrument_agent_agent") + _process_module_definition( + "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" + ) + + _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") + _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") _process_module_definition( "mcp.server.fastmcp.tools.tool_manager", diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 92867d1b83..1e87ddddb0 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,7 +163,6 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) - # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py new file mode 100644 index 0000000000..8bb977808f --- /dev/null +++ b/newrelic/hooks/mlmodel_strands.py @@ -0,0 +1,408 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid + +from newrelic.api.error_trace import ErrorTraceWrapper +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.core.context import ContextOf + +_logger = logging.getLogger(__name__) +STRANDS_VERSION = get_package_version("strands-agents") + +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." +TOOL_OUTPUT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record output of tool call. Please report this issue to New Relic Support." + + +def wrap_agent__call__(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + # Make a copy of the invocation state before we mutate it + if "invocation_state" in bound_args: + invocation_state = bound_args["invocation_state"] = dict(bound_args["invocation_state"] or {}) + + # Attempt to save the current transaction context into the invocation state dictionary + invocation_state["_nr_transaction"] = trace + except Exception: + return wrapped(*args, **kwargs) + else: + return wrapped(**bound_args) + + +async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): + # If there's already a transaction, don't propagate anything here + if current_transaction(): + return await wrapped(*args, **kwargs) + + try: + # Grab the trace context we should be running under and pass it to ContextOf + bound_args = bind_args(wrapped, args, kwargs) + invocation_state = bound_args["invocation_state"] or {} + trace = invocation_state.pop("_nr_transaction", None) + except Exception: + return await wrapped(*args, **kwargs) + + # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + if trace: + with ContextOf(trace=trace): + return await wrapped(*args, **kwargs) + else: + return await wrapped(*args, **kwargs) + + +def wrap_stream_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + func_name = callable_name(wrapped) + agent_name = getattr(instance, "name", "agent") + function_trace_name = f"{func_name}/{agent_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/agent/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + agent_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_agent_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} + return proxied_return_val + + +def _record_agent_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _record_tool_event_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + + try: + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + return + + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks and duplicate reporting + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, linking_metadata): + try: + try: + tool_output = tool_results[-1]["content"][0] if tool_results else None + error = tool_results[-1]["status"] == "error" + except Exception: + tool_output = None + error = False + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + tool_name = strands_attrs.get("tool_name", "tool") + tool_id = strands_attrs.get("tool_id", None) + run_id = strands_attrs.get("run_id", None) + tool_input = strands_attrs.get("tool_input", None) + agent_name = strands_attrs.get("agent_name", "agent") + settings = transaction.settings or global_settings() + + tool_event_dict = { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "agent_name": agent_name, + "vendor": "strands", + "ingest_source": "Python", + } + # Set error flag if the status shows an error was caught, + # it will be reported further down in the instrumentation. + if error: + tool_event_dict["error"] = True + + if settings.ai_monitoring.record_content.enabled: + tool_event_dict.update({"input": tool_input}) + tool_event_dict.update({"output": tool_output}) + tool_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + tool_event_dict = {} + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + + return tool_event_dict + + +def _handle_agent_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + agent_name = strands_attrs.get("agent_name", "agent") + agent_id = strands_attrs.get("agent_id", None) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"agent_id": agent_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + agent_event_dict = _construct_base_agent_event_dict( + agent_name, agent_id, transaction, linking_metadata, "strands" + ) + agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) + transaction.record_custom_event("LlmAgent", agent_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def _handle_tool_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + strands_attrs = getattr(self, "_nr_strands_attrs", {}) + + # If there are no strands attrs exit early as there's no data to record. + if not strands_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + + # Use saved linking metadata to maintain correct span association + linking_metadata = self._nr_metadata or get_trace_linking_metadata() + + try: + tool_id = strands_attrs["tool_id"] + + # We expect this to never have any output since this is an error case, + # but if it does we will report it. + try: + tool_results = strands_attrs.get("tool_results", []) + except Exception: + tool_results = None + _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) + + # Notice the error on the function trace + self._nr_ft.notice_error(attributes={"tool_id": tool_id}) + self._nr_ft.__exit__(*sys.exc_info()) + + # Create error event + tool_event_dict = _construct_base_tool_event_dict( + strands_attrs, tool_results, transaction, linking_metadata, "strands" + ) + tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + transaction.record_custom_event("LlmTool", tool_event_dict) + + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE, exc_info=True) + finally: + # Clear cached data to prevent memory leaks + if hasattr(self, "_nr_strands_attrs"): + self._nr_strands_attrs.clear() + + +def wrap_tool_executor__stream(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("Strands", STRANDS_VERSION) + transaction._add_agent_attribute("llm", True) + + # Grab tool data + bound_args = bind_args(wrapped, args, kwargs) + agent_name = getattr(bound_args.get("agent"), "name", "agent") + tool_use = bound_args.get("tool_use", {}) + + run_id = tool_use.get("toolUseId", "") + tool_name = tool_use.get("name", "tool") + _input = tool_use.get("input") + tool_input = str(_input) if _input else None + tool_results = bound_args.get("tool_results", []) + + func_name = callable_name(wrapped) + function_trace_name = f"{func_name}/{tool_name}" + + ft = FunctionTrace(name=function_trace_name, group="Llm/tool/Strands") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + tool_id = str(uuid.uuid4()) + + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _handle_tool_streaming_completion_error(ft, transaction, exc) + raise + + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = { + "tool_results": tool_results, + "tool_name": tool_name, + "tool_id": tool_id, + "run_id": run_id, + "tool_input": tool_input, + "agent_name": agent_name, + } + return proxied_return_val + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped, on_stop_iteration, on_error): + super().__init__(wrapped) + self._nr_on_stop_iteration = on_stop_iteration + self._nr_on_error = on_error + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + except StopAsyncIteration: + self._nr_on_stop_iteration(self, transaction) + raise + except Exception as exc: + self._nr_on_error(self, transaction, exc) + raise + return return_val + + async def aclose(self): + return await super().aclose() + + +def wrap_ToolRegister_register_tool(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + bound_args["tool"]._tool_func = ErrorTraceWrapper(bound_args["tool"]._tool_func) + return wrapped(*args, **kwargs) + + +def instrument_agent_agent(module): + if hasattr(module, "Agent"): + if hasattr(module.Agent, "__call__"): # noqa: B004 + wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) + if hasattr(module.Agent, "invoke_async"): + wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) + if hasattr(module.Agent, "_run_loop"): + wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) + + +def instrument_tools_executors__executor(module): + if hasattr(module, "ToolExecutor"): + if hasattr(module.ToolExecutor, "_stream"): + wrap_function_wrapper(module, "ToolExecutor._stream", wrap_tool_executor__stream) + + +def instrument_tools_registry(module): + if hasattr(module, "ToolRegistry"): + if hasattr(module.ToolRegistry, "register_tool"): + wrap_function_wrapper(module, "ToolRegistry.register_tool", wrap_ToolRegister_register_tool) diff --git a/tests/mlmodel_strands/conftest.py b/tests/mlmodel_strands/conftest.py index b810161f6a..a2ad9b8dd0 100644 --- a/tests/mlmodel_strands/conftest.py +++ b/tests/mlmodel_strands/conftest.py @@ -14,6 +14,7 @@ import pytest from _mock_model_provider import MockedModelProvider +from testing_support.fixture.event_loop import event_loop as loop from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture from testing_support.ml_testing_utils import set_trace_info @@ -50,15 +51,33 @@ def single_tool_model(): @pytest.fixture -def single_tool_model_error(): +def single_tool_model_runtime_error_coro(): model = MockedModelProvider( [ { "role": "assistant", "content": [ - {"text": "Calling add_exclamation tool"}, + {"text": "Calling throw_exception_coro tool"}, + # Set arguments to an invalid type to trigger error in tool + {"toolUse": {"name": "throw_exception_coro", "toolUseId": "123", "input": {"message": "Hello"}}}, + ], + }, + {"role": "assistant", "content": [{"text": "Success!"}]}, + ] + ) + return model + + +@pytest.fixture +def single_tool_model_runtime_error_agen(): + model = MockedModelProvider( + [ + { + "role": "assistant", + "content": [ + {"text": "Calling throw_exception_agen tool"}, # Set arguments to an invalid type to trigger error in tool - {"toolUse": {"name": "add_exclamation", "toolUseId": "123", "input": {"message": 12}}}, + {"toolUse": {"name": "throw_exception_agen", "toolUseId": "123", "input": {"message": "Hello"}}}, ], }, {"role": "assistant", "content": [{"text": "Success!"}]}, diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py deleted file mode 100644 index ae24003fab..0000000000 --- a/tests/mlmodel_strands/test_simple.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from strands import Agent, tool - -from newrelic.api.background_task import background_task - - -# Example tool for testing purposes -@tool -def add_exclamation(message: str) -> str: - return f"{message}!" - - -# TODO: Remove this file once all real tests are in place - - -@background_task() -def test_simple_run_agent(set_trace_info, single_tool_model): - set_trace_info() - my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) - - response = my_agent("Run the tools.") - assert response.message["content"][0]["text"] == "Success!" - assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 3d93e06e30..540e44f70c 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -797,7 +797,7 @@ def _bind_params(transaction, *args, **kwargs): transaction = _bind_params(*args, **kwargs) error_events = transaction.error_events(instance.stats_table) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for sample in error_events: assert isinstance(sample, list) assert len(sample) == 3 diff --git a/tests/testing_support/validators/validate_custom_event.py b/tests/testing_support/validators/validate_custom_event.py index deeef7fb25..c3cf78032a 100644 --- a/tests/testing_support/validators/validate_custom_event.py +++ b/tests/testing_support/validators/validate_custom_event.py @@ -61,7 +61,9 @@ def _validate_custom_event_count(wrapped, instance, args, kwargs): raise else: stats = core_application_stats_engine(None) - assert stats.custom_events.num_samples == count + assert stats.custom_events.num_samples == count, ( + f"Expected: {count}, Got: {stats.custom_events.num_samples}. Events: {list(stats.custom_events)}" + ) return result diff --git a/tests/testing_support/validators/validate_error_event_collector_json.py b/tests/testing_support/validators/validate_error_event_collector_json.py index d1cec3a558..27ea76f3a3 100644 --- a/tests/testing_support/validators/validate_error_event_collector_json.py +++ b/tests/testing_support/validators/validate_error_event_collector_json.py @@ -52,7 +52,7 @@ def _validate_error_event_collector_json(wrapped, instance, args, kwargs): error_events = decoded_json[2] - assert len(error_events) == num_errors + assert len(error_events) == num_errors, f"Expected: {num_errors}, Got: {len(error_events)}" for event in error_events: # event is an array containing intrinsics, user-attributes, # and agent-attributes diff --git a/tests/testing_support/validators/validate_transaction_error_event_count.py b/tests/testing_support/validators/validate_transaction_error_event_count.py index b41a52330f..f5e8c0b206 100644 --- a/tests/testing_support/validators/validate_transaction_error_event_count.py +++ b/tests/testing_support/validators/validate_transaction_error_event_count.py @@ -28,7 +28,9 @@ def _validate_error_event_on_stats_engine(wrapped, instance, args, kwargs): raise else: error_events = list(instance.error_events) - assert len(error_events) == num_errors + assert len(error_events) == num_errors, ( + f"Expected: {num_errors}, Got: {len(error_events)}. Errors: {error_events}" + ) return result From 0bfe25d19530884e7aaf4db7fe9a01e968ebdb12 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:11:49 -0700 Subject: [PATCH 17/31] Add tool and agent instrumentation. --- newrelic/common/llm_utils.py | 21 ------------------- newrelic/config.py | 1 - newrelic/hooks/mlmodel_strands.py | 35 +++++++++++++++++++++++-------- 3 files changed, 26 insertions(+), 31 deletions(-) diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 0ae8575477..2ec1136e6d 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -12,11 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - -_logger = logging.getLogger(__name__) -RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Autogen instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" - def _get_llm_metadata(transaction): # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events @@ -28,19 +23,3 @@ def _get_llm_metadata(transaction): return llm_metadata_dict - -def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata, vendor_name): - try: - agent_event_dict = { - "id": agent_id, - "name": agent_name, - "span_id": linking_metadata.get("span.id"), - "trace_id": linking_metadata.get("trace.id"), - "vendor": vendor_name, - "ingest_source": "Python", - } - agent_event_dict.update(_get_llm_metadata(transaction)) - except Exception: - agent_event_dict = {} - - return agent_event_dict diff --git a/newrelic/config.py b/newrelic/config.py index 806563a1f6..09a82e7c9a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2950,7 +2950,6 @@ def _process_module_builtin_defaults(): _process_module_definition( "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" ) - _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 8bb977808f..6ac032d8e6 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -20,7 +20,7 @@ from newrelic.api.function_trace import FunctionTrace from newrelic.api.time_trace import current_trace, get_trace_linking_metadata from newrelic.api.transaction import current_transaction -from newrelic.common.llm_utils import _construct_base_agent_event_dict, _get_llm_metadata +from newrelic.common.llm_utils import _get_llm_metadata from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version @@ -67,7 +67,7 @@ async def wrap_agent_invoke_async(wrapped, instance, args, kwargs): except Exception: return await wrapped(*args, **kwargs) - # If we found a transaction to propagate, use it. Otherwise, just call wrapped. + # If we find a transaction to propagate, use it. Otherwise, just call wrapped. if trace: with ContextOf(trace=trace): return await wrapped(*args, **kwargs) @@ -129,7 +129,7 @@ def _record_agent_event_on_stop_iteration(self, transaction): agent_name = strands_attrs.get("agent_name", "agent") agent_id = strands_attrs.get("agent_id", None) agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -209,6 +209,7 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li if settings.ai_monitoring.record_content.enabled: tool_event_dict.update({"input": tool_input}) + # In error cases, the output will hold the error message tool_event_dict.update({"output": tool_output}) tool_event_dict.update(_get_llm_metadata(transaction)) except Exception: @@ -218,7 +219,23 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li return tool_event_dict -def _handle_agent_streaming_completion_error(self, transaction, exc): +def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata): + try: + agent_event_dict = { + "id": agent_id, + "name": agent_name, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "strands", + "ingest_source": "Python", + } + agent_event_dict.update(_get_llm_metadata(transaction)) + except Exception: + agent_event_dict = {} + + return agent_event_dict + +def _handle_agent_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -240,7 +257,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): # Create error event agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata, "strands" + agent_name, agent_id, transaction, linking_metadata ) agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -253,7 +270,7 @@ def _handle_agent_streaming_completion_error(self, transaction, exc): self._nr_strands_attrs.clear() -def _handle_tool_streaming_completion_error(self, transaction, exc): +def _handle_tool_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -282,7 +299,7 @@ def _handle_tool_streaming_completion_error(self, transaction, exc): # Create error event tool_event_dict = _construct_base_tool_event_dict( - strands_attrs, tool_results, transaction, linking_metadata, "strands" + strands_attrs, tool_results, transaction, linking_metadata ) tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmTool", tool_event_dict) @@ -372,7 +389,7 @@ async def __anext__(self): self._nr_on_stop_iteration(self, transaction) raise except Exception as exc: - self._nr_on_error(self, transaction, exc) + self._nr_on_error(self, transaction) raise return return_val @@ -392,7 +409,7 @@ def instrument_agent_agent(module): wrap_function_wrapper(module, "Agent.__call__", wrap_agent__call__) if hasattr(module.Agent, "invoke_async"): wrap_function_wrapper(module, "Agent.invoke_async", wrap_agent_invoke_async) - if hasattr(module.Agent, "_run_loop"): + if hasattr(module.Agent, "stream_async"): wrap_function_wrapper(module, "Agent.stream_async", wrap_stream_async) From b757886bcf2bfe26a948ad8a69c42994263235a3 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:12:06 -0700 Subject: [PATCH 18/31] Add tests file. --- tests/mlmodel_strands/test_agent.py | 372 ++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100644 tests/mlmodel_strands/test_agent.py diff --git a/tests/mlmodel_strands/test_agent.py b/tests/mlmodel_strands/test_agent.py new file mode 100644 index 0000000000..e2bc83b9dd --- /dev/null +++ b/tests/mlmodel_strands/test_agent.py @@ -0,0 +1,372 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from strands import Agent, tool +from testing_support.fixtures import reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_with_context_attrs, + tool_events_sans_content, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import validate_error_trace_attributes +from testing_support.validators.validate_transaction_error_event_count import validate_transaction_error_event_count +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import transient_function_wrapper + +tool_recorded_event = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "output": "{'text': 'Hello!'}", + "name": "add_exclamation", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_coro = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_coro", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +tool_recorded_event_error_agen = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "throw_exception_agen", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "output": "{'text': 'Error: RuntimeError - Oops'}", + "duration": None, + }, + ) +] + + +agent_recorded_event = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + }, + ) +] + +agent_recorded_event_error = [ + ( + {"type": "LlmAgent"}, + { + "id": None, + "name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "vendor": "strands", + "ingest_source": "Python", + "error": True, + "duration": None, + }, + ) +] + + +# Example tool for testing purposes +@tool +async def add_exclamation(message: str) -> str: + return f"{message}!" + + +@tool +async def throw_exception_coro(message: str) -> str: + raise RuntimeError("Oops") + + +@tool +async def throw_exception_agen(message: str) -> str: + raise RuntimeError("Oops") + yield + + +@reset_core_stats_engine() +@validate_custom_events(events_with_context_attrs(tool_recorded_event)) +@validate_custom_events(events_with_context_attrs(agent_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + with WithLlmCustomAttributes({"context": "attr"}): + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = await my_agent.invoke_async('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@validate_custom_events(tool_recorded_event) +@validate_custom_events(agent_recorded_event) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_stream_async", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_stream_async(loop, set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + async def _test(): + response = my_agent.stream_async('Add an exclamation to the word "Hello"') + messages = [event["message"]["content"] async for event in response if "message" in event] + + assert len(messages) == 3 + assert messages[0][0]["text"] == "Calling add_exclamation tool" + assert messages[0][1]["toolUse"]["name"] == "add_exclamation" + assert messages[1][0]["toolResult"]["content"][0]["text"] == "Hello!" + assert messages[2][0]["text"] == "Success!" + + loop.run_until_complete(_test()) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +@validate_custom_events(agent_recorded_event) +@validate_custom_events(tool_events_sans_content(tool_recorded_event)) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_no_content", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_no_content(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task() +def test_agent_invoke_disabled_ai_monitoring_events(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event_error) +@validate_custom_event_count(count=1) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_error", + scoped_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + rollup_metrics=[("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1)], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_error(set_trace_info, single_tool_model): + # Add a wrapper to intentionally force an error in the Agent code + @transient_function_wrapper("strands.agent.agent", "Agent._convert_prompt_to_messages") + def _wrap_convert_prompt_to_messages(wrapped, instance, args, kwargs): + raise ValueError("Oops") + + @_wrap_convert_prompt_to_messages + def _test(): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + my_agent('Add an exclamation to the word "Hello"') # raises ValueError + + with pytest.raises(ValueError): + _test() + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_coro) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_coro_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_coro", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_coro_runtime_error(set_trace_info, single_tool_model_runtime_error_coro): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_coro, tools=[throw_exception_coro]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_coro"].error_count == 1 + + +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(RuntimeError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(tool_recorded_event_error_agen) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_invoke_tool_agen_runtime_error", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/throw_exception_agen", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_invoke_tool_agen_runtime_error(set_trace_info, single_tool_model_runtime_error_agen): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model_runtime_error_agen, tools=[throw_exception_agen]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["throw_exception_agen"].error_count == 1 + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_agent_invoke_outside_txn(single_tool_model): + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent('Add an exclamation to the word "Hello"') + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 From 4667155da58586e420dd8df9554b86448fabe928 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 13:32:24 -0700 Subject: [PATCH 19/31] Cleanup instrumentation. --- newrelic/hooks/mlmodel_strands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 6ac032d8e6..5d17952ff1 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -99,8 +99,7 @@ def wrap_stream_async(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_agent_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata @@ -346,8 +345,7 @@ def wrap_tool_executor__stream(wrapped, instance, args, kwargs): try: return_val = wrapped(*args, **kwargs) - except Exception as exc: - _handle_tool_streaming_completion_error(ft, transaction, exc) + except Exception: raise # For streaming responses, wrap with proxy and attach metadata From cec6515b8e87c9e4eed43d45f98fd880b98c5921 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 14:15:52 -0700 Subject: [PATCH 20/31] Cleanup. Co-authored-by: Tim Pansino --- newrelic/api/time_trace.py | 1 + newrelic/hooks/datastore_elasticsearch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index 7d2ad59a8b..fd0f62fdef 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,6 +361,7 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} + # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 1e87ddddb0..92867d1b83 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,6 +163,7 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) + # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is From bc841dfacb5c33f3542fdf8a1bd730f814c77ea6 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 15:07:56 -0700 Subject: [PATCH 21/31] Handle additional args in mock model. --- tests/mlmodel_strands/_mock_model_provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mlmodel_strands/_mock_model_provider.py b/tests/mlmodel_strands/_mock_model_provider.py index e4c9e79930..ef60e13bad 100644 --- a/tests/mlmodel_strands/_mock_model_provider.py +++ b/tests/mlmodel_strands/_mock_model_provider.py @@ -41,7 +41,7 @@ def __init__(self, agent_responses): def format_chunk(self, event): return event - def format_request(self, messages, tool_specs=None, system_prompt=None): + def format_request(self, messages, tool_specs=None, system_prompt=None, **kwargs): return None def get_config(self): @@ -53,7 +53,7 @@ def update_config(self, **model_config): async def structured_output(self, output_model, prompt, system_prompt=None, **kwargs): pass - async def stream(self, messages, tool_specs=None, system_prompt=None): + async def stream(self, messages, tool_specs=None, system_prompt=None, **kwargs): events = self.map_agent_message_to_events(self.agent_responses[self.index]) for event in events: yield event From 78c7bd681db00460e7e6ffa7a8ab426fe24f03cf Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 23 Oct 2025 14:32:16 -0700 Subject: [PATCH 22/31] Strands Mock Model (#1551) * Add strands to tox.ini * Add mock models for strands testing * Add simple test file to validate strands mocking --- tests/mlmodel_strands/test_simple.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py new file mode 100644 index 0000000000..ae24003fab --- /dev/null +++ b/tests/mlmodel_strands/test_simple.py @@ -0,0 +1,36 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from strands import Agent, tool + +from newrelic.api.background_task import background_task + + +# Example tool for testing purposes +@tool +def add_exclamation(message: str) -> str: + return f"{message}!" + + +# TODO: Remove this file once all real tests are in place + + +@background_task() +def test_simple_run_agent(set_trace_info, single_tool_model): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + + response = my_agent("Run the tools.") + assert response.message["content"][0]["text"] == "Success!" + assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 From 3863fdf7a070637bd53578a48532c0d214ba322b Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 27 Oct 2025 12:44:35 -0700 Subject: [PATCH 23/31] Add baseline instrumentation. --- newrelic/api/time_trace.py | 1 - newrelic/common/llm_utils.py | 3 +- newrelic/hooks/datastore_elasticsearch.py | 1 - newrelic/hooks/mlmodel_strands.py | 16 +++++----- tests/mlmodel_strands/test_simple.py | 36 ----------------------- 5 files changed, 10 insertions(+), 47 deletions(-) delete mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index fd0f62fdef..7d2ad59a8b 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,7 +361,6 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} - # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 2ec1136e6d..16bb046b80 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -21,5 +21,4 @@ def _get_llm_metadata(transaction): if llm_context_attrs: llm_metadata_dict.update(llm_context_attrs) - return llm_metadata_dict - + return llm_metadata_dict \ No newline at end of file diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 92867d1b83..1e87ddddb0 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,7 +163,6 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) - # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 5d17952ff1..4a46dc7cdd 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -1,3 +1,4 @@ + # Copyright 2010 New Relic, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -127,9 +128,7 @@ def _record_agent_event_on_stop_iteration(self, transaction): agent_name = strands_attrs.get("agent_name", "agent") agent_id = strands_attrs.get("agent_id", None) - agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata - ) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -234,6 +233,7 @@ def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_ return agent_event_dict + def _handle_agent_streaming_completion_error(self, transaction): if hasattr(self, "_nr_ft"): strands_attrs = getattr(self, "_nr_strands_attrs", {}) @@ -255,9 +255,7 @@ def _handle_agent_streaming_completion_error(self, transaction): self._nr_ft.__exit__(*sys.exc_info()) # Create error event - agent_event_dict = _construct_base_agent_event_dict( - agent_name, agent_id, transaction, linking_metadata - ) + agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) agent_event_dict.update({"duration": self._nr_ft.duration * 1000, "error": True}) transaction.record_custom_event("LlmAgent", agent_event_dict) @@ -301,6 +299,10 @@ def _handle_tool_streaming_completion_error(self, transaction): strands_attrs, tool_results, transaction, linking_metadata ) tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + # Ensure error flag is set to True in case the tool_results did not indicate an error + if "error" not in tool_event_dict: + tool_event_dict.update({"error": True}) + transaction.record_custom_event("LlmTool", tool_event_dict) except Exception: @@ -386,7 +388,7 @@ async def __anext__(self): except StopAsyncIteration: self._nr_on_stop_iteration(self, transaction) raise - except Exception as exc: + except Exception: self._nr_on_error(self, transaction) raise return return_val diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py deleted file mode 100644 index ae24003fab..0000000000 --- a/tests/mlmodel_strands/test_simple.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from strands import Agent, tool - -from newrelic.api.background_task import background_task - - -# Example tool for testing purposes -@tool -def add_exclamation(message: str) -> str: - return f"{message}!" - - -# TODO: Remove this file once all real tests are in place - - -@background_task() -def test_simple_run_agent(set_trace_info, single_tool_model): - set_trace_info() - my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) - - response = my_agent("Run the tools.") - assert response.message["content"][0]["text"] == "Success!" - assert response.metrics.tool_metrics["add_exclamation"].success_count == 1 From 30ab602cb06dfc9dc5b6a032f503fcdd11e345d1 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 28 Oct 2025 23:11:49 -0700 Subject: [PATCH 24/31] Add tool and agent instrumentation. --- newrelic/hooks/mlmodel_strands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 4a46dc7cdd..d09129b80d 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -1,4 +1,3 @@ - # Copyright 2010 New Relic, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); From b4a657860d90d874a77fc05851c3fd3627b53678 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 30 Oct 2025 14:15:52 -0700 Subject: [PATCH 25/31] Cleanup. Co-authored-by: Tim Pansino --- newrelic/api/time_trace.py | 1 + newrelic/hooks/datastore_elasticsearch.py | 1 + 2 files changed, 2 insertions(+) diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index 7d2ad59a8b..fd0f62fdef 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -361,6 +361,7 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c def notice_error(self, error=None, attributes=None, expected=None, ignore=None, status_code=None): attributes = attributes if attributes is not None else {} + # If no exception details provided, use current exception. # Pull from sys.exc_info if no exception is passed diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 1e87ddddb0..92867d1b83 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -163,6 +163,7 @@ async def _nr_wrapper_AsyncElasticsearch_method_(wrapped, instance, args, kwargs if transaction is None: return await wrapped(*args, **kwargs) + # When index is None, it means there is no target field # associated with this method. Hence this method will only # create an operation metric and no statement metric. This is From 2b8107e7a731e7c2e1ba937727271ab127002980 Mon Sep 17 00:00:00 2001 From: umaannamalai <19895951+umaannamalai@users.noreply.github.com> Date: Thu, 30 Oct 2025 21:19:34 +0000 Subject: [PATCH 26/31] [MegaLinter] Apply linters fixes --- newrelic/common/llm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/common/llm_utils.py b/newrelic/common/llm_utils.py index 16bb046b80..eebdacfc7f 100644 --- a/newrelic/common/llm_utils.py +++ b/newrelic/common/llm_utils.py @@ -21,4 +21,4 @@ def _get_llm_metadata(transaction): if llm_context_attrs: llm_metadata_dict.update(llm_context_attrs) - return llm_metadata_dict \ No newline at end of file + return llm_metadata_dict From 5527059718eb3cb431a7a8afb2ba71d936fbe2f1 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 3 Nov 2025 19:12:11 -0800 Subject: [PATCH 27/31] Add test to force exception and exercise _handle_tool_streaming_completion_error. --- tests/mlmodel_strands/test_agent.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/mlmodel_strands/test_agent.py b/tests/mlmodel_strands/test_agent.py index e2bc83b9dd..af685668ad 100644 --- a/tests/mlmodel_strands/test_agent.py +++ b/tests/mlmodel_strands/test_agent.py @@ -51,6 +51,24 @@ ) ] +tool_recorded_event_forced_internal_error = [ + ( + {"type": "LlmTool"}, + { + "id": None, + "run_id": "123", + "name": "add_exclamation", + "agent_name": "my_agent", + "span_id": None, + "trace_id": "trace-id", + "input": "{'message': 'Hello'}", + "vendor": "strands", + "ingest_source": "Python", + "duration": None, + "error": True, + }, + ) +] tool_recorded_event_error_coro = [ ( @@ -362,6 +380,43 @@ def test_agent_invoke_tool_agen_runtime_error(set_trace_info, single_tool_model_ assert response.metrics.tool_metrics["throw_exception_agen"].error_count == 1 +@reset_core_stats_engine() +@validate_transaction_error_event_count(1) +@validate_error_trace_attributes(callable_name(ValueError), exact_attrs={"agent": {}, "intrinsic": {}, "user": {}}) +@validate_custom_events(agent_recorded_event) +@validate_custom_events(tool_recorded_event_forced_internal_error) +@validate_custom_event_count(count=2) +@validate_transaction_metrics( + "test_agent:test_agent_tool_forced_exception", + scoped_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + rollup_metrics=[ + ("Llm/agent/Strands/strands.agent.agent:Agent.stream_async/my_agent", 1), + ("Llm/tool/Strands/strands.tools.executors._executor:ToolExecutor._stream/add_exclamation", 1), + ], + background_task=True, +) +@validate_attributes("agent", ["llm"]) +@background_task() +def test_agent_tool_forced_exception(set_trace_info, single_tool_model): + # Add a wrapper to intentionally force an error in the ToolExecutor._stream code to hit the exception path in + # the AsyncGeneratorProxy + @transient_function_wrapper("strands.hooks.events", "BeforeToolCallEvent.__init__") + def _wrap_BeforeToolCallEvent_init(wrapped, instance, args, kwargs): + raise ValueError("Oops") + + @_wrap_BeforeToolCallEvent_init + def _test(): + set_trace_info() + my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) + my_agent('Add an exclamation to the word "Hello"') + + # This will not explicitly raise a ValueError when running the test but we are still able to capture it in the error trace + _test() + + @reset_core_stats_engine() @validate_custom_event_count(count=0) def test_agent_invoke_outside_txn(single_tool_model): From 43bc7922787b568ec306e93298d075e9f2dbe423 Mon Sep 17 00:00:00 2001 From: Tim Pansino Date: Thu, 6 Nov 2025 16:16:42 -0800 Subject: [PATCH 28/31] Implement strands context passing instrumentation. --- newrelic/config.py | 1 + newrelic/hooks/mlmodel_strands.py | 50 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index 09a82e7c9a..ff2d85e359 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2951,6 +2951,7 @@ def _process_module_builtin_defaults(): "strands.tools.executors._executor", "newrelic.hooks.mlmodel_strands", "instrument_tools_executors__executor" ) _process_module_definition("strands.tools.registry", "newrelic.hooks.mlmodel_strands", "instrument_tools_registry") + _process_module_definition("strands.models.bedrock", "newrelic.hooks.mlmodel_strands", "instrument_models_bedrock") _process_module_definition("mcp.client.session", "newrelic.hooks.adapter_mcp", "instrument_mcp_client_session") _process_module_definition( diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index d09129b80d..df50e5a3de 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -402,6 +402,47 @@ def wrap_ToolRegister_register_tool(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) +def wrap_bedrock_model_stream(wrapped, instance, args, kwargs): + """Stores trace context on the messages argument to be retrieved by the _stream() instrumentation.""" + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + settings = trace.settings or global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + try: + bound_args = bind_args(wrapped, args, kwargs) + except Exception: + return wrapped(*args, **kwargs) + + if "messages" in bound_args and isinstance(bound_args["messages"], list): + bound_args["messages"].append({"newrelic_trace": trace}) + + return wrapped(*args, **kwargs) + + +def wrap_bedrock_model__stream(wrapped, instance, args, kwargs): + """Retrieves trace context stored on the messages argument and propagates it to the new thread.""" + try: + bound_args = bind_args(wrapped, args, kwargs) + except Exception: + return wrapped(*args, **kwargs) + + if ( + "messages" in bound_args + and isinstance(bound_args["messages"], list) + and bound_args["messages"] # non-empty list + and "newrelic_trace" in bound_args["messages"][-1] + ): + trace_message = bound_args["messages"].pop() + with ContextOf(trace=trace_message["newrelic_trace"]): + return wrapped(*args, **kwargs) + + return wrapped(*args, **kwargs) + + def instrument_agent_agent(module): if hasattr(module, "Agent"): if hasattr(module.Agent, "__call__"): # noqa: B004 @@ -422,3 +463,12 @@ def instrument_tools_registry(module): if hasattr(module, "ToolRegistry"): if hasattr(module.ToolRegistry, "register_tool"): wrap_function_wrapper(module, "ToolRegistry.register_tool", wrap_ToolRegister_register_tool) + + +def instrument_models_bedrock(module): + # This instrumentation only exists to pass trace context due to bedrock models using a separate thread. + if hasattr(module, "BedrockModel"): + if hasattr(module.BedrockModel, "stream"): + wrap_function_wrapper(module, "BedrockModel.stream", wrap_bedrock_model_stream) + if hasattr(module.BedrockModel, "_stream"): + wrap_function_wrapper(module, "BedrockModel._stream", wrap_bedrock_model__stream) From 79ada63b6e85beec9da0c8a47242cc6084354430 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 11 Nov 2025 13:41:22 -0800 Subject: [PATCH 29/31] Address review feedback. --- newrelic/api/error_trace.py | 6 ++ newrelic/hooks/mlmodel_strands.py | 105 ++++++++++++++++++------------ 2 files changed, 68 insertions(+), 43 deletions(-) diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index c94ed34dc9..aaa12b50e3 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -46,17 +46,23 @@ def __exit__(self, exc, value, tb): def ErrorTraceWrapper(wrapped, ignore=None, expected=None, status_code=None, async_wrapper=None): def literal_wrapper(wrapped, instance, args, kwargs): + # Determine if the wrapped function is async or sync wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + # Sync function path if not wrapper: parent = current_trace() if not parent: + # No active tracing context so just call the wrapped function directly return wrapped(*args, **kwargs) + # Async function path else: + # For async functions, the async wrapper will handle trace context propagation parent = None trace = ErrorTrace(ignore, expected, status_code, parent=parent) if wrapper: + # The async wrapper handles the context management for us return wrapper(wrapped, trace)(*args, **kwargs) with trace: diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index df50e5a3de..8694ded289 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -33,6 +33,9 @@ RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record LLM events. Please report this issue to New Relic Support." TOOL_OUTPUT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record output of tool call. Please report this issue to New Relic Support." +AGENT_EVENT_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to record agent data. Please report this issue to New Relic Support." +TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to extract tool information. If the issue persists, report this issue to New Relic support.\n" + def wrap_agent__call__(wrapped, instance, args, kwargs): @@ -103,13 +106,19 @@ def wrap_stream_async(wrapped, instance, args, kwargs): raise # For streaming responses, wrap with proxy and attach metadata - proxied_return_val = AsyncGeneratorProxy( - return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error - ) - proxied_return_val._nr_ft = ft - proxied_return_val._nr_metadata = linking_metadata - proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} - return proxied_return_val + try: + # For streaming responses, wrap with proxy and attach metadata + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_agent_event_on_stop_iteration, _handle_agent_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = {"agent_name": agent_name, "agent_id": agent_id} + return proxied_return_val + except Exception: + # If proxy creation fails, clean up the function trace and return original value + ft.__exit__(*sys.exc_info()) + return return_val def _record_agent_event_on_stop_iteration(self, transaction): @@ -126,9 +135,9 @@ def _record_agent_event_on_stop_iteration(self, transaction): return agent_name = strands_attrs.get("agent_name", "agent") - agent_id = strands_attrs.get("agent_id", None) + agent_id = strands_attrs.get("agent_id") agent_event_dict = _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_metadata) - agent_event_dict.update({"duration": self._nr_ft.duration * 1000}) + agent_event_dict["duration"] = self._nr_ft.duration * 1000 transaction.record_custom_event("LlmAgent", agent_event_dict) except Exception: @@ -161,7 +170,7 @@ def _record_tool_event_on_stop_iteration(self, transaction): tool_event_dict = _construct_base_tool_event_dict( strands_attrs, tool_results, transaction, linking_metadata ) - tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + tool_event_dict["duration"] = self._nr_ft.duration * 1000 transaction.record_custom_event("LlmTool", tool_event_dict) except Exception: @@ -183,9 +192,9 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li _logger.warning(TOOL_OUTPUT_FAILURE_LOG_MESSAGE, exc_info=True) tool_name = strands_attrs.get("tool_name", "tool") - tool_id = strands_attrs.get("tool_id", None) - run_id = strands_attrs.get("run_id", None) - tool_input = strands_attrs.get("tool_input", None) + tool_id = strands_attrs.get("tool_id") + run_id = strands_attrs.get("run_id") + tool_input = strands_attrs.get("tool_input") agent_name = strands_attrs.get("agent_name", "agent") settings = transaction.settings or global_settings() @@ -205,9 +214,9 @@ def _construct_base_tool_event_dict(strands_attrs, tool_results, transaction, li tool_event_dict["error"] = True if settings.ai_monitoring.record_content.enabled: - tool_event_dict.update({"input": tool_input}) + tool_event_dict["input"] = tool_input # In error cases, the output will hold the error message - tool_event_dict.update({"output": tool_output}) + tool_event_dict["output"] = tool_output tool_event_dict.update(_get_llm_metadata(transaction)) except Exception: tool_event_dict = {} @@ -228,6 +237,7 @@ def _construct_base_agent_event_dict(agent_name, agent_id, transaction, linking_ } agent_event_dict.update(_get_llm_metadata(transaction)) except Exception: + _logger.warning(AGENT_EVENT_FAILURE_LOG_MESSAGE, exc_info=True) agent_event_dict = {} return agent_event_dict @@ -247,7 +257,7 @@ def _handle_agent_streaming_completion_error(self, transaction): try: agent_name = strands_attrs.get("agent_name", "agent") - agent_id = strands_attrs.get("agent_id", None) + agent_id = strands_attrs.get("agent_id") # Notice the error on the function trace self._nr_ft.notice_error(attributes={"agent_id": agent_id}) @@ -279,7 +289,7 @@ def _handle_tool_streaming_completion_error(self, transaction): linking_metadata = self._nr_metadata or get_trace_linking_metadata() try: - tool_id = strands_attrs["tool_id"] + tool_id = strands_attrs.get("tool_id") # We expect this to never have any output since this is an error case, # but if it does we will report it. @@ -297,10 +307,10 @@ def _handle_tool_streaming_completion_error(self, transaction): tool_event_dict = _construct_base_tool_event_dict( strands_attrs, tool_results, transaction, linking_metadata ) - tool_event_dict.update({"duration": self._nr_ft.duration * 1000}) + tool_event_dict["duration"] = self._nr_ft.duration * 1000 # Ensure error flag is set to True in case the tool_results did not indicate an error if "error" not in tool_event_dict: - tool_event_dict.update({"error": True}) + tool_event_dict["error"] = True transaction.record_custom_event("LlmTool", tool_event_dict) @@ -326,15 +336,19 @@ def wrap_tool_executor__stream(wrapped, instance, args, kwargs): transaction._add_agent_attribute("llm", True) # Grab tool data - bound_args = bind_args(wrapped, args, kwargs) - agent_name = getattr(bound_args.get("agent"), "name", "agent") - tool_use = bound_args.get("tool_use", {}) - - run_id = tool_use.get("toolUseId", "") - tool_name = tool_use.get("name", "tool") - _input = tool_use.get("input") - tool_input = str(_input) if _input else None - tool_results = bound_args.get("tool_results", []) + try: + bound_args = bind_args(wrapped, args, kwargs) + agent_name = getattr(bound_args.get("agent"), "name", "agent") + tool_use = bound_args.get("tool_use", {}) + + run_id = tool_use.get("toolUseId", "") + tool_name = tool_use.get("name", "tool") + _input = tool_use.get("input") + tool_input = str(_input) if _input else None + tool_results = bound_args.get("tool_results", []) + except Exception: + tool_name = "tool" + _logger.warning(TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE, exc_info=True) func_name = callable_name(wrapped) function_trace_name = f"{func_name}/{tool_name}" @@ -349,21 +363,26 @@ def wrap_tool_executor__stream(wrapped, instance, args, kwargs): except Exception: raise - # For streaming responses, wrap with proxy and attach metadata - proxied_return_val = AsyncGeneratorProxy( - return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error - ) - proxied_return_val._nr_ft = ft - proxied_return_val._nr_metadata = linking_metadata - proxied_return_val._nr_strands_attrs = { - "tool_results": tool_results, - "tool_name": tool_name, - "tool_id": tool_id, - "run_id": run_id, - "tool_input": tool_input, - "agent_name": agent_name, - } - return proxied_return_val + try: + # Wrap return value with proxy and attach metadata for later access + proxied_return_val = AsyncGeneratorProxy( + return_val, _record_tool_event_on_stop_iteration, _handle_tool_streaming_completion_error + ) + proxied_return_val._nr_ft = ft + proxied_return_val._nr_metadata = linking_metadata + proxied_return_val._nr_strands_attrs = { + "tool_results": tool_results, + "tool_name": tool_name, + "tool_id": tool_id, + "run_id": run_id, + "tool_input": tool_input, + "agent_name": agent_name, + } + return proxied_return_val + except Exception: + # If proxy creation fails, clean up the function trace and return original value + ft.__exit__(*sys.exc_info()) + return return_val class AsyncGeneratorProxy(ObjectProxy): From 80e9d3ddde5f6a6e2e1903c100df5b15599f168e Mon Sep 17 00:00:00 2001 From: umaannamalai <19895951+umaannamalai@users.noreply.github.com> Date: Tue, 11 Nov 2025 21:42:57 +0000 Subject: [PATCH 30/31] [MegaLinter] Apply linters fixes --- newrelic/hooks/mlmodel_strands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic/hooks/mlmodel_strands.py b/newrelic/hooks/mlmodel_strands.py index 8694ded289..bf849fd717 100644 --- a/newrelic/hooks/mlmodel_strands.py +++ b/newrelic/hooks/mlmodel_strands.py @@ -37,7 +37,6 @@ TOOL_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in Strands instrumentation: Failed to extract tool information. If the issue persists, report this issue to New Relic support.\n" - def wrap_agent__call__(wrapped, instance, args, kwargs): trace = current_trace() if not trace: From 36d4af2d9209dba15372b1f65009bfc52cfbd3a8 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Wed, 12 Nov 2025 13:12:46 -0800 Subject: [PATCH 31/31] Remove test_simple.py file. --- tests/mlmodel_strands/test_simple.py | 36 ---------------------------- 1 file changed, 36 deletions(-) delete mode 100644 tests/mlmodel_strands/test_simple.py diff --git a/tests/mlmodel_strands/test_simple.py b/tests/mlmodel_strands/test_simple.py deleted file mode 100644 index ae24003fab..0000000000 --- a/tests/mlmodel_strands/test_simple.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from strands import Agent, tool - -from newrelic.api.background_task import background_task - - -# Example tool for testing purposes -@tool -def add_exclamation(message: str) -> str: - return f"{message}!" - - -# TODO: Remove this file once all real tests are in place - - -@background_task() -def test_simple_run_agent(set_trace_info, single_tool_model): - set_trace_info() - my_agent = Agent(name="my_agent", model=single_tool_model, tools=[add_exclamation]) - - response = my_agent("Run the tools.") - assert response.message["content"][0]["text"] == "Success!" - assert response.metrics.tool_metrics["add_exclamation"].success_count == 1