[Higher Level API] Added decorators for better dev UX and a Chatbot application (#19)

Adding the Aimon Rely README, images, the postman collection, a simple client and examples.

A few small changes for error handling in the client and the example application.

Getting the Aimon API key from the streamlit app

updating README

Updating langchain example gif

Updating API endpoint

Adding V2 API with support for conciseness, completeness and toxicity checks (#1)

* Adding V2 API with support for conciseness, completeness and toxicity checks.

* Removing prints and updating config for the example application.

* Updating README

---------

Co-authored-by: Preetam Joshi <info@aimon.ai>

Updating postman collection

Fixed the simple aimon client's handling of batch requests.

Updated postman collection. Added support for a user_query parameter in the input data dictionary.

Updating readme

Fixed bug in the example app

Uploading client code

Adding more convenience APIs

Fixing bug in create_dataset

Added Github actions config to publish to PyPI. Cleaned up dependencies and updated documentation.

Fixing langchain example

Fixing doc links

Formatting changes

Changes for aimon-rely

* Adding instruction adherence and hallucination v0.2 to the client

Updating git ignore

Adding more to gitignore

Removing .idea files

* Fixing doc string

* Updating documentation

* Updating Client to use V3 API

* Fixing test

* Updating tests

* Updating documentation in the client

* Adding .streamlit dir to .gitignore

* initial version of decorators for syntactic sugar

* A few more changes

* updating analyze and detect decorators

* Adding new notebooks

* Fixing bug in analyze decorator

---------

Co-authored-by: Preetam Joshi <info@aimon.ai>

Author: pjoshi30

.gitignore

@@ -22,3 +22,5 @@ lib64
 
 # Installer logs
 pip-log.txt
+
+.streamlit/*

aimon/__init__.py

@@ -62,7 +62,7 @@
     "DEFAULT_MAX_RETRIES",
     "DEFAULT_CONNECTION_LIMITS",
     "DefaultHttpxClient",
-    "DefaultAsyncHttpxClient",
+    "DefaultAsyncHttpxClient"
 ]
 
 _setup_logging()
@@ -79,3 +79,6 @@
         except (TypeError, AttributeError):
             # Some of our exported symbols are builtins which we can't set attributes for.
             pass
+
+from .decorators.detect import DetectWithContextQuery, DetectWithContextQueryInstructions, DetectWithQueryFuncReturningContext, DetectWithQueryInstructionsFuncReturningContext
+from .decorators.analyze import Analyze, Application, Model

aimon/decorators/__init__.py

@@ -0,0 +1,123 @@
+from functools import wraps
+from .common import AimonClientSingleton
+
+
+class Application:
+    def __init__(self, name, stage="evaluation", type="text", metadata={}):
+        self.name = name
+        self.stage = stage
+        self.type = type
+        self.metadata = metadata
+
+
+class Model:
+    def __init__(self, name, model_type, metadata={}):
+        self.name = name
+        self.model_type = model_type
+        self.metadata = metadata
+
+
+class Analyze(object):
+    DEFAULT_CONFIG = {'hallucination': {'detector_name': 'default'}}
+
+    def __init__(self, application, model, api_key=None, evaluation_name=None, dataset_collection_name=None, eval_tags=None, instructions=None, config=None):
+        self.client = AimonClientSingleton.get_instance(api_key)
+        self.application = application
+        self.model = model
+        self.evaluation_name = evaluation_name
+        self.dataset_collection_name = dataset_collection_name
+        self.eval_tags = eval_tags
+        self.instructions = instructions
+        self.config = config if config else self.DEFAULT_CONFIG
+        self.initialize()
+
+    def initialize(self):
+        # Create or retrieve the model
+        self._am_model = self.client.models.create(
+            name=self.model.name,
+            type=self.model.model_type,
+            description="This model is named {} and is of type {}".format(self.model.name, self.model.model_type),
+            metadata=self.model.metadata
+        )
+
+        # Create or retrieve the application
+        self._am_app = self.client.applications.create(
+            name=self.application.name,
+            model_name=self._am_model.name,
+            stage=self.application.stage,
+            type=self.application.type,
+            metadata=self.application.metadata
+        )
+
+        if self.evaluation_name is not None:
+
+            if self.dataset_collection_name is None:
+                raise ValueError("Dataset collection name must be provided for running an evaluation.")
+
+            # Create or retrieve the dataset collection
+            self._am_dataset_collection = self.client.datasets.collection.retrieve(name=self.dataset_collection_name)
+
+            # Create or retrieve the evaluation
+            self._eval = self.client.evaluations.create(
+                name=self.evaluation_name,
+                application_id=self._am_app.id,
+                model_id=self._am_model.id,
+                dataset_collection_id=self._am_dataset_collection.id
+            )
+
+    def _run_eval(self, func, args, kwargs):
+        # Create an evaluation run
+        eval_run = self.client.evaluations.run.create(
+            evaluation_id=self._eval.id,
+            metrics_config=self.config,
+        )
+        # Get all records from the datasets
+        dataset_collection_records = []
+        for dataset_id in self._am_dataset_collection.dataset_ids:
+            dataset_records = self.client.datasets.records.list(sha=dataset_id)
+            dataset_collection_records.extend(dataset_records)
+        results = []
+        for record in dataset_collection_records:
+            result = func(record['context_docs'], record['user_query'], *args, **kwargs)
+            payload = {
+                "application_id": self._am_app.id,
+                "version": self._am_app.version,
+                "prompt": record['prompt'] or "",
+                "user_query": record['user_query'] or "",
+                "context_docs": [d for d in record['context_docs']],
+                "output": result,
+                "evaluation_id": self._eval.id,
+                "evaluation_run_id": eval_run.id,
+            }
+            results.append((result, self.client.analyze.create(body=[payload])))
+        return results
+
+    def _run_production_analysis(self, func, context, sys_prompt, user_query, args, kwargs):
+        result = func(context, sys_prompt, user_query, *args, **kwargs)
+        if result is None:
+            raise ValueError("Result must be returned by the decorated function")
+        payload = {
+            "application_id": self._am_app.id,
+            "version": self._am_app.version,
+            "prompt": sys_prompt or "",
+            "user_query": user_query or "",
+            "context_docs": context or [],
+            "output": result
+        }
+        aimon_response = self.client.analyze.create(body=[payload])
+        return aimon_response, result
+
+    def __call__(self, func):
+        @wraps(func)
+        def wrapper(context=None, sys_prompt=None, user_query=None, *args, **kwargs):
+
+            if self.evaluation_name is not None:
+                return self._run_eval(func, args, kwargs)
+            else:
+                # Production mode, run the provided args through the user function
+                return self._run_production_analysis(func, context, sys_prompt, user_query, args, kwargs)
+
+        return wrapper
+
+
+analyze = Analyze

aimon/decorators/analyze.py

@@ -0,0 +1,15 @@
+import os
+from aimon import Client
+
+
+# A singleton class that instantiates the Aimon client once
+# and provides a method to get the client instance
+class AimonClientSingleton:
+    _instance = None
+
+    @staticmethod
+    def get_instance(api_key=None):
+        if AimonClientSingleton._instance is None:
+            api_key = os.getenv('AIMON_API_KEY') if not api_key else api_key
+            AimonClientSingleton._instance = Client(auth_header="Bearer {}".format(api_key))
+        return AimonClientSingleton._instance

aimon/decorators/common.py

@@ -0,0 +1,107 @@
+from functools import wraps
+
+from .common import AimonClientSingleton
+
+
+class DetectWithQueryFuncReturningContext(object):
+    DEFAULT_CONFIG = {'hallucination': {'detector_name': 'default'}}
+
+    def __init__(self, api_key=None, config=None):
+        self.client = AimonClientSingleton.get_instance(api_key)
+        self.config = config if config else self.DEFAULT_CONFIG
+
+    def __call__(self, func):
+        @wraps(func)
+        def wrapper(user_query, *args, **kwargs):
+            result, context = func(user_query, *args, **kwargs)
+
+            if result is None or context is None:
+                raise ValueError("Result and context must be returned by the decorated function")
+
+            data_to_send = [{
+                "user_query": user_query,
+                "context": context,
+                "generated_text": result,
+                "config": self.config
+            }]
+
+            aimon_response = self.client.inference.detect(body=data_to_send)[0]
+            return result, context, aimon_response
+
+        return wrapper
+
+
+class DetectWithQueryInstructionsFuncReturningContext(DetectWithQueryFuncReturningContext):
+    def __call__(self, func):
+        @wraps(func)
+        def wrapper(user_query, instructions, *args, **kwargs):
+            result, context = func(user_query, instructions, *args, **kwargs)
+
+            if result is None or context is None:
+                raise ValueError("Result and context must be returned by the decorated function")
+
+            data_to_send = [{
+                "user_query": user_query,
+                "context": context,
+                "generated_text": result,
+                "instructions": instructions,
+                "config": self.config
+            }]
+
+            aimon_response = self.client.inference.detect(body=data_to_send)[0]
+            return result, context, aimon_response
+
+        return wrapper
+
+
+# Another class but does not include instructions in the wrapper call
+class DetectWithContextQuery(object):
+    DEFAULT_CONFIG = {'hallucination': {'detector_name': 'default'}}
+
+    def __init__(self, api_key=None, config=None):
+        self.client = AimonClientSingleton.get_instance(api_key)
+        self.config = config if config else self.DEFAULT_CONFIG
+
+    def __call__(self, func):
+        @wraps(func)
+        def wrapper(context, user_query, *args, **kwargs):
+            result = func(context, user_query, *args, **kwargs)
+
+            if result is None:
+                raise ValueError("Result must be returned by the decorated function")
+
+            data_to_send = [{
+                "context": context,
+                "user_query": user_query,
+                "generated_text": result,
+                "config": self.config
+            }]
+
+            aimon_response = self.client.inference.detect(body=data_to_send)[0]
+            return result, aimon_response
+
+        return wrapper
+
+
+class DetectWithContextQueryInstructions(DetectWithContextQuery):
+    def __call__(self, func):
+        @wraps(func)
+        def wrapper(context, user_query, instructions, *args, **kwargs):
+            result = func(context, user_query, instructions, *args, **kwargs)
+
+            if result is None:
+                raise ValueError("Result must be returned by the decorated function")
+
+            data_to_send = [{
+                "context": context,
+                "user_query": user_query,
+                "generated_text": result,
+                "instructions": instructions,
+                "config": self.config
+            }]
+
+            aimon_response = self.client.inference.detect(body=data_to_send)[0]
+            return result, aimon_response
+
+        return wrapper
+