Add AdapterParseError to dspy (#8212)

* Add AdapterParseError to dspy

* change __init__.py

* fix test

Author: chenmoneygithub

dspy/adapters/chat_adapter.py

@@ -16,6 +16,7 @@
 from dspy.clients.lm import LM
 from dspy.signatures.signature import Signature
 from dspy.utils.callback import BaseCallback
+from dspy.utils.exceptions import AdapterParseError
 
 field_header_pattern = re.compile(r"\[\[ ## (\w+) ## \]\]")
 
@@ -168,11 +169,19 @@ def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]:
                 try:
                     fields[k] = parse_value(v, signature.output_fields[k].annotation)
                 except Exception as e:
-                    raise ValueError(
-                        f"Error parsing field {k}: {e}.\n\n\t\tOn attempting to parse the value\n```\n{v}\n```"
+                    raise AdapterParseError(
+                        adapter_name="ChatAdapter",
+                        signature=signature,
+                        lm_response=completion,
+                        message=f"Failed to parse field {k} with value {v} from the LM response. Error message: {e}",
                     )
         if fields.keys() != signature.output_fields.keys():
-            raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+            raise AdapterParseError(
+                adapter_name="ChatAdapter",
+                signature=signature,
+                lm_response=completion,
+                parsed_result=fields,
+            )
 
         return fields
 

dspy/adapters/json_adapter.py

@@ -18,6 +18,7 @@
 )
 from dspy.clients.lm import LM
 from dspy.signatures.signature import Signature, SignatureMeta
+from dspy.utils.exceptions import AdapterParseError
 
 logger = logging.getLogger(__name__)
 
@@ -66,7 +67,11 @@ def __call__(
             try:
                 lm_kwargs["response_format"] = {"type": "json_object"}
                 return super().__call__(lm, lm_kwargs, signature, demos, inputs)
+            except AdapterParseError as e:
+                # On AdapterParseError, we raise the original error.
+                raise e
             except Exception as e:
+                # On any other error, we raise a RuntimeError with the original error message.
                 raise RuntimeError(
                     "Both structured output format and JSON mode failed. Please choose a model that supports "
                     f"`response_format` argument. Original error: {e}"
@@ -124,7 +129,12 @@ def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]:
         fields = json_repair.loads(completion)
 
         if not isinstance(fields, dict):
-            raise ValueError(f"Expected a JSON object but parsed a {type(fields)}")
+            raise AdapterParseError(
+                adapter_name="JSONAdapter",
+                signature=signature,
+                lm_response=completion,
+                message="LM response cannot be serialized to a JSON object.",
+            )
 
         fields = {k: v for k, v in fields.items() if k in signature.output_fields}
 
@@ -134,7 +144,12 @@ def parse(self, signature: Type[Signature], completion: str) -> dict[str, Any]:
                 fields[k] = parse_value(v, signature.output_fields[k].annotation)
 
         if fields.keys() != signature.output_fields.keys():
-            raise ValueError(f"Expected {signature.output_fields.keys()} but got {fields.keys()}")
+            raise AdapterParseError(
+                adapter_name="JSONAdapter",
+                signature=signature,
+                lm_response=completion,
+                parsed_result=fields,
+            )
 
         return fields
 

dspy/utils/__init__.py

@@ -1,6 +1,7 @@
 from dspy.utils.callback import BaseCallback, with_callbacks
 from dspy.utils.dummies import DummyLM, DummyVectorizer, dummy_rm
 from dspy.streaming.messages import StatusMessageProvider, StatusMessage
+from dspy.utils import exceptions
 
 import os
 import requests
@@ -20,6 +21,7 @@ def download(url):
 
 __all__ = [
     "download",
+    "exceptions",
     "BaseCallback",
     "with_callbacks",
     "DummyLM",

dspy/utils/exceptions.py

@@ -0,0 +1,33 @@
+from typing import Optional
+
+from dspy.signatures.signature import Signature
+
+
+class AdapterParseError(Exception):
+    """Exception raised when adapter cannot parse the LM response."""
+
+    def __init__(
+        self,
+        adapter_name: str,
+        signature: Signature,
+        lm_response: str,
+        message: Optional[str] = None,
+        parsed_result: Optional[str] = None,
+    ):
+        self.adapter_name = adapter_name
+        self.signature = signature
+        self.lm_response = lm_response
+        self.parsed_result = parsed_result
+
+        message = f"{message}\n\n" if message else ""
+        message = (
+            f"{message}"
+            f"Adapter {adapter_name} failed to parse the LM response. \n\n"
+            f"LM Response: {lm_response} \n\n"
+            f"Expected to find output fields in the LM response: [{', '.join(signature.output_fields.keys())}] \n\n"
+        )
+
+        if parsed_result is not None:
+            message += f"Actual output fields parsed from the LM response: [{', '.join(parsed_result.keys())}] \n\n"
+
+        super().__init__(message)

tests/adapters/test_json_adapter.py

@@ -3,6 +3,7 @@
 import pydantic
 import pytest
 from pydantic import create_model
+from litellm.utils import ModelResponse, Message, Choices
 
 import dspy
 
@@ -195,9 +196,28 @@ class TestSignature(dspy.Signature):
 
 
 def test_json_adapter_parse_raise_error_on_mismatch_fields():
-    signature = dspy.make_signature("input1->output1")
+    signature = dspy.make_signature("question->answer")
     adapter = dspy.JSONAdapter()
-    invalid_completion = '{"output": "Test output"}'
-    with pytest.raises(ValueError) as error:
-        adapter.parse(signature, invalid_completion)
-    assert str(error.value) == "Expected dict_keys(['output1']) but got dict_keys([])"
+
+    with mock.patch("litellm.completion") as mock_completion:
+        mock_completion.return_value = ModelResponse(
+            choices=[
+                Choices(message=Message(content="{'answer1': 'Paris'}")),
+            ],
+            model="openai/gpt4o",
+        )
+        lm = dspy.LM(model="openai/gpt-4o-mini")
+        with pytest.raises(dspy.utils.exceptions.AdapterParseError) as e:
+            adapter(lm, {}, signature, [], {"question": "What is the capital of France?"})
+
+    assert e.value.adapter_name == "JSONAdapter"
+    assert e.value.signature == signature
+    assert e.value.lm_response == "{'answer1': 'Paris'}"
+    assert e.value.parsed_result == {}
+
+    assert str(e.value) == (
+        "Adapter JSONAdapter failed to parse the LM response. \n\n"
+        "LM Response: {'answer1': 'Paris'} \n\n"
+        "Expected to find output fields in the LM response: [answer] \n\n"
+        "Actual output fields parsed from the LM response: [] \n\n"
+    )

tests/utils/test_exceptions.py

@@ -0,0 +1,63 @@
+import pytest
+from dspy.utils.exceptions import AdapterParseError
+from dspy.signatures.signature import Signature
+import dspy
+
+
+def test_adapter_parse_error_basic():
+    adapter_name = "ChatAdapter"
+    signature = dspy.make_signature("question->answer1, answer2")
+    lm_response = "[[ ## answer1 ## ]]\nanswer1"
+
+    error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response)
+
+    assert error.adapter_name == adapter_name
+    assert error.signature == signature
+    assert error.lm_response == lm_response
+
+    error_message = str(error)
+    assert error_message == (
+        "Adapter ChatAdapter failed to parse the LM response. \n\n"
+        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+    )
+
+
+def test_adapter_parse_error_with_message():
+    adapter_name = "ChatAdapter"
+    signature = dspy.make_signature("question->answer1, answer2")
+    lm_response = "[[ ## answer1 ## ]]\nanswer1"
+    message = "Critical error, please fix!"
+
+    error = AdapterParseError(adapter_name=adapter_name, signature=signature, lm_response=lm_response, message=message)
+
+    assert error.adapter_name == adapter_name
+    assert error.signature == signature
+    assert error.lm_response == lm_response
+
+    error_message = str(error)
+    assert error_message == (
+        "Critical error, please fix!\n\n"
+        "Adapter ChatAdapter failed to parse the LM response. \n\n"
+        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+    )
+
+
+def test_adapter_parse_error_with_parsed_result():
+    adapter_name = "ChatAdapter"
+    signature = dspy.make_signature("question->answer1, answer2")
+    lm_response = "[[ ## answer1 ## ]]\nanswer1"
+    parsed_result = {"answer1": "value1"}
+
+    error = AdapterParseError(
+        adapter_name=adapter_name, signature=signature, lm_response=lm_response, parsed_result=parsed_result
+    )
+
+    error_message = str(error)
+    assert error_message == (
+        "Adapter ChatAdapter failed to parse the LM response. \n\n"
+        "LM Response: [[ ## answer1 ## ]]\nanswer1 \n\n"
+        "Expected to find output fields in the LM response: [answer1, answer2] \n\n"
+        "Actual output fields parsed from the LM response: [answer1] \n\n"
+    )