Merge branch 'main' of https://github.com/drawal1/dspy into main

Author: drawal1

README.md

@@ -146,6 +146,8 @@ You can find other examples tweeted by [@lateinteraction](https://twitter.com/la
   - [Using DSPy to train Gpt 3.5 on HumanEval by Thomas Ahle](https://github.com/stanfordnlp/dspy/blob/main/examples/functional/functional.ipynb)
   - [Building a chess playing agent using DSPy by Franck SN](https://medium.com/thoughts-on-machine-learning/building-a-chess-playing-agent-using-dspy-9b87c868f71e)
 
+TODO: Add links to the state-of-the-art results on Theory of Mind (ToM) by Plastic Labs, the results by Haize Labs for Red Teaming with DSPy, and the DSPy pipeline from Replit.
+
 There are also recent cool examples at [Weaviate's DSPy cookbook](https://github.com/weaviate/recipes/tree/main/integrations/dspy) by Connor Shorten. [See tutorial on YouTube](https://www.youtube.com/watch?v=CEuUG4Umfxs).
 
 ## 3) Syntax: You're in charge of the workflow—it's free-form Python code!

docs/api/language_model_clients/aws_models.md

@@ -2,7 +2,7 @@
 sidebar_position: 9
 ---
 
-# dsp.AWSMistral, dsp.AWSAnthropic, dsp.AWSMeta
+# dspy.AWSMistral, dspy.AWSAnthropic, dspy.AWSMeta
 
 ### Usage
 
@@ -83,4 +83,4 @@ Refer to [`dspy.OpenAI`](https://dspy-docs.vercel.app/api/language_model_clients
 
 <br/>
 
-`AWSAnthropic` and `AWSMeta` work exactly the same as `AWSMistral`.
+`AWSAnthropic` and `AWSMeta` work exactly the same as `AWSMistral`.

dsp/modules/aws_providers.py

@@ -8,13 +8,13 @@ class AWSProvider(ABC):
     """This abstract class adds support for AWS model providers such as Bedrock and SageMaker.
     The subclasses such as Bedrock and Sagemaker implement the abstract method _call_model and work in conjunction with the AWSModel classes.
     Usage Example:
-        bedrock = Bedrock(region_name="us-west-2")
-        bedrock_mixtral = AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
-        bedrock_haiku = AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
-        bedrock_llama2 = AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
+        bedrock = dspy.Bedrock(region_name="us-west-2")
+        bedrock_mixtral = dspy.AWSMistral(bedrock, "mistral.mixtral-8x7b-instruct-v0:1", **kwargs)
+        bedrock_haiku = dspy.AWSAnthropic(bedrock, "anthropic.claude-3-haiku-20240307-v1:0", **kwargs)
+        bedrock_llama2 = dspy.AWSMeta(bedrock, "meta.llama2-13b-chat-v1", **kwargs)
 
-        sagemaker = Sagemaker(region_name="us-west-2")
-        sagemaker_model = AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
+        sagemaker = dspy.Sagemaker(region_name="us-west-2")
+        sagemaker_model = dspy.AWSMistral(sagemaker, "<YOUR_ENDPOINT_NAME>", **kwargs)
     """
 
     def __init__(

dsp/modules/azure_openai.py

@@ -44,7 +44,7 @@ class AzureOpenAI(LM):
     Args:
         api_base (str): Azure URL endpoint for model calling, often called 'azure_endpoint'.
         api_version (str): Version identifier for API.
-        model (str, optional): OpenAI or Azure supported LLM model to use. Defaults to "text-davinci-002".
+        model (str, optional): OpenAI or Azure supported LLM model to use. Defaults to "gpt-3.5-turbo-instruct".
         api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.
         model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "chat".
         **kwargs: Additional arguments to pass to the API provider.

dsp/modules/gpt3.py

@@ -40,7 +40,7 @@ class GPT3(LM):
     """Wrapper around OpenAI's GPT API.
 
     Args:
-        model (str, optional): OpenAI supported LLM model to use. Defaults to "text-davinci-002".
+        model (str, optional): OpenAI supported LLM model to use. Defaults to "gpt-3.5-turbo-instruct".
         api_key (Optional[str], optional): API provider Authentication token. use Defaults to None.
         api_provider (Literal["openai"], optional): The API provider to use. Defaults to "openai".
         model_type (Literal["chat", "text"], optional): The type of model that was specified. Mainly to decide the optimal prompting strategy. Defaults to "text".

dsp/modules/lm.py

@@ -46,7 +46,7 @@ def inspect_history(self, n: int = 1, skip: int = 0):
             prompt = x["prompt"]
 
             if prompt != last_prompt:
-                if provider == "clarifai" or provider == "google" or provider == "Bedrock" or provider == "Sagemaker":
+                if provider == "clarifai" or provider == "google" or provider == "groq" or provider == "Bedrock" or provider == "Sagemaker":
                     printed.append((prompt, x["response"]))
                 elif provider == "anthropic":
                     blocks = [{"text": block.text} for block in x["response"].content if block.type == "text"]

dspy/evaluate/evaluate.py

@@ -166,16 +166,17 @@ def wrapped_program(example_idx, example):
                 num_threads,
                 display_progress,
             )
-        if return_outputs:  # Handle the return_outputs logic
-            results = [(example, prediction, score)
-                       for _, example, prediction, score in reordered_devset]
 
         if display:
             print(
                 f"Average Metric: {ncorrect} / {ntotal}  ({round(100 * ncorrect / ntotal, 1)}%)")
 
         predicted_devset = sorted(reordered_devset)
 
+        if return_outputs:  # Handle the return_outputs logic
+            results = [(example, prediction, score)
+                       for _, example, prediction, score in predicted_devset]
+
         # data = [{**example, **prediction, 'correct': score} for example, prediction, score in zip(reordered_devset, preds, scores)]
         data = [
             merge_dicts(example, prediction) | {"correct": score} for _, example, prediction, score in predicted_devset
@@ -222,9 +223,9 @@ def wrapped_program(example_idx, example):
                 ipython_display(HTML(message))
 
         if return_all_scores and return_outputs:
-            return round(100 * ncorrect / ntotal, 2), results, [score for *_, score in reordered_devset]
+            return round(100 * ncorrect / ntotal, 2), results, [score for *_, score in predicted_devset]
         elif return_all_scores:
-            return round(100 * ncorrect / ntotal, 2), [score for *_, score in reordered_devset]
+            return round(100 * ncorrect / ntotal, 2), [score for *_, score in predicted_devset]
         elif return_outputs:
             return round(100 * ncorrect / ntotal, 2), results
 

dspy/functional/functional.py

@@ -14,18 +14,32 @@
 from dspy.signatures.signature import ensure_signature, make_signature
 
 
-def predictor(func) -> dspy.Module:
-    """Decorator that creates a predictor module based on the provided function."""
-    signature = _func_to_signature(func)
-    *_, output_key = signature.output_fields.keys()
-    return _StripOutput(TypedPredictor(signature), output_key)
-
-
-def cot(func) -> dspy.Module:
-    """Decorator that creates a chain of thought module based on the provided function."""
-    signature = _func_to_signature(func)
-    *_, output_key = signature.output_fields.keys()
-    return _StripOutput(TypedChainOfThought(signature), output_key)
+def predictor(*args, **kwargs):
+    def _predictor(func) -> dspy.Module:
+        """Decorator that creates a predictor module based on the provided function."""
+        signature = _func_to_signature(func)
+        *_, output_key = signature.output_fields.keys()
+        return _StripOutput(TypedPredictor(signature, **kwargs), output_key)
+
+    # if we have only a single callable argument, the decorator was invoked with no key word arguments
+    #  so we just return the wrapped function 
+    if len(args) == 1 and callable(args[0]) and len(kwargs) == 0:
+        return _predictor(args[0])
+    return _predictor
+
+
+def cot(*args, **kwargs):
+    def _cot(func) -> dspy.Module:
+        """Decorator that creates a chain of thought module based on the provided function."""
+        signature = _func_to_signature(func)
+        *_, output_key = signature.output_fields.keys()
+        return _StripOutput(TypedChainOfThought(signature, **kwargs), output_key)
+
+    # if we have only a single callable argument, the decorator was invoked with no key word arguments
+    #  so we just return the wrapped function 
+    if len(args) == 1 and callable(args[0]) and len(kwargs) == 0:
+        return _cot(args[0])
+    return _cot
 
 
 class _StripOutput(dspy.Module):

dspy/retrieve/qdrant_rm.py

@@ -53,7 +53,7 @@ def __init__(
 
         super().__init__(k=k)
 
-    def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int],**kwargs) -> dspy.Prediction:
+    def forward(self, query_or_queries: Union[str, List[str]], k: Optional[int] = None,**kwargs) -> dspy.Prediction:
         """Search with Qdrant for self.k top passages for query
 
         Args:

pyproject.toml

@@ -36,6 +36,7 @@ dependencies = [
     "requests",
     "optuna",
     "pydantic>=2.5.0,<=2.7",
+    "structlog"
 ]
 
 [project.optional-dependencies]