Bicep for GPT-4

Author: pamelafox

.devcontainer/devcontainer.json

@@ -19,7 +19,9 @@
         "ghcr.io/azure/azure-dev/azd:latest": {
 			"version": "1.10.1"
 		},
-		"ghcr.io/prulloac/devcontainer-features/ollama:1": {}
+		"ghcr.io/prulloac/devcontainer-features/ollama:1": {},
+		// az feature
+		"ghcr.io/devcontainers/features/azure-cli": {}
     },
 	// Configure tool-specific properties.
 	"customizations": {

.env.sample

@@ -13,12 +13,15 @@ OPENAI_EMBED_HOST=azure
 # You also need to `azd auth login` if running this locally
 AZURE_OPENAI_ENDPOINT=https://YOUR-AZURE-OPENAI-SERVICE-NAME.openai.azure.com
 AZURE_OPENAI_VERSION=2024-03-01-preview
-AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-35-turbo
-AZURE_OPENAI_CHAT_MODEL=gpt-35-turbo
+AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini
+AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini
 AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-ada-002
 AZURE_OPENAI_EMBED_MODEL=text-embedding-ada-002
-AZURE_OPENAI_EMBED_MODEL_DIMENSIONS=1536
+AZURE_OPENAI_EMBED_DIMENSIONS=1536
 AZURE_OPENAI_EMBEDDING_COLUMN=embedding_ada002
+AZURE_OPENAI_EVAL_DEPLOYMENT=gpt-4
+AZURE_OPENAI_EVAL_MODEL=gpt-4
+AZURE_TENANT_ID=
 # Only needed when using key-based Azure authentication:
 AZURE_OPENAI_KEY=
 # Needed for OpenAI.com:

.github/workflows/evaluate.yaml

@@ -2,6 +2,11 @@ name: Evaluate
 
 on:
   workflow_dispatch:
+  # TODO: change later
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 # Set up permissions for deploying with secretless Azure federated credentials
 # https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux#set-up-azure-login-with-openid-connect-authentication
@@ -17,15 +22,20 @@ jobs:
       AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
       AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
-      AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
-      AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
-      AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_4_EVAL_DEPLOYMENT_NAME }}
-      AZURE_OPENAI_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_DEPLOYMENT_NAME }}
-      AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME: ${{ vars.AZURE_OPENAI_35_TURBO_DEPLOYMENT_NAME }}
       AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
-      AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
-      AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
-
+      OPENAI_CHAT_HOST: ${{ vars.OPENAI_CHAT_HOST }}
+      OPENAI_EMBED_HOST: ${{ vars.OPENAI_EMBED_HOST }}
+      AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
+      AZURE_OPENAI_VERSION: ${{ vars.AZURE_OPENAI_VERSION }}
+      AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
+      AZURE_OPENAI_CHAT_MODEL: ${{ vars.AZURE_OPENAI_CHAT_MODEL }}
+      AZURE_OPENAI_EMBED_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBED_DEPLOYMENT }}
+      AZURE_OPENAI_EMBED_MODEL: ${{ vars.AZURE_OPENAI_EMBED_MODEL }}
+      AZURE_OPENAI_EMBED_DIMENSIONS: ${{ vars.AZURE_OPENAI_EMBED_DIMENSIONS }}
+      AZURE_OPENAI_EMBEDDING_COLUMN: ${{ vars.AZURE_OPENAI_EMBEDDING_COLUMN }}
+      AZURE_OPENAI_EVAL_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EVAL_DEPLOYMENT }}
+      AZURE_OPENAI_EVAL_MODEL: ${{ vars.AZURE_OPENAI_EVAL_MODEL }}
+      # Postgres environment variables are not needed, as they're passed in below
     steps:
       - uses: actions/checkout@v4
       - name: Check for MacOS Runner

azure.yaml

@@ -39,12 +39,19 @@ hooks:
 pipeline:
   variables:
     - DEPLOY_AZURE_OPENAI
-    - AZURE_OPENAI_CHAT_MODEL
+    - OPENAI_CHAT_HOST
+    - OPENAI_EMBED_HOST
+    - AZURE_OPENAI_ENDPOINT
+    - AZURE_OPENAI_VERSION
     - AZURE_OPENAI_CHAT_DEPLOYMENT
+    - AZURE_OPENAI_CHAT_MODEL
     - AZURE_OPENAI_CHAT_DEPLOYMENT_VERSION
     - AZURE_OPENAI_CHAT_DEPLOYMENT_CAPACITY
-    - AZURE_OPENAI_EMBED_MODEL
     - AZURE_OPENAI_EMBED_DEPLOYMENT
+    - AZURE_OPENAI_EMBED_MODEL
     - AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION
     - AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY
     - AZURE_OPENAI_EMBED_DIMENSIONS
+    - AZURE_OPENAI_EMBEDDING_COLUMN
+    - AZURE_OPENAI_EVAL_DEPLOYMENT
+    - AZURE_OPENAI_EVAL_MODEL

docs/evaluation.md

@@ -1,5 +1,30 @@
 # Evaluating the RAG answer quality
 
+## Deploy a GPT-4 model
+
+
+1. Run this command to tell `azd` to deploy a GPT-4 model for evaluation:
+
+    ```shell
+    azd env set DEPLOY_EVAL_MODEL true
+    ```
+
+2. Set the capacity to the highest possible value to ensure that the evaluation runs quickly.
+
+    ```shell
+    azd env set AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY 100
+    ```
+
+    By default, that will provision a `gpt-4` model, version `turbo-2024-04-09`. To change those settings, set the `AZURE_OPENAI_EVAL_DEPLOYMENT` and `AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION` environment variables.
+
+3. Then, run the following command to provision the model:
+
+    ```shell
+    azd provision
+    ```
+
+## Setup the evaluation environment
+
 Install all the dependencies for the evaluation script by running the following command:
 
 ```bash

evals/evaluate.py

@@ -2,47 +2,54 @@
 import os
 from pathlib import Path
 
+import azure.identity
+from azure.ai.evaluation import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from dotenv import load_dotenv
 from evaltools.eval.evaluate import run_evaluate_from_config
-from promptflow.core import AzureOpenAIModelConfiguration, ModelConfiguration, OpenAIModelConfiguration
+from rich.logging import RichHandler
 
 logger = logging.getLogger("ragapp")
 
 
-def get_openai_config() -> ModelConfiguration:
+def get_openai_config() -> dict:
     if os.environ.get("OPENAI_CHAT_HOST") == "azure":
         azure_endpoint = os.environ["AZURE_OPENAI_ENDPOINT"]
-        azure_deployment = os.environ.get("AZURE_OPENAI_CHAT_DEPLOYMENT")
-        api_version = "2023-07-01-preview"
+        azure_deployment = os.environ.get("AZURE_OPENAI_EVAL_DEPLOYMENT")
         if os.environ.get("AZURE_OPENAI_KEY"):
             logger.info("Using Azure OpenAI Service with API Key from AZURE_OPENAI_KEY")
-            openai_config = AzureOpenAIModelConfiguration(
-                azure_endpoint=azure_endpoint,
-                azure_deployment=azure_deployment,
-                api_version=api_version,
-                api_key=os.environ["AZURE_OPENAI_KEY"],
-            )
+            openai_config: AzureOpenAIModelConfiguration = {
+                "azure_endpoint": azure_endpoint,
+                "azure_deployment": azure_deployment,
+                "api_key": os.environ["AZURE_OPENAI_KEY"],
+            }
         else:
-            logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential")
-            openai_config = AzureOpenAIModelConfiguration(
-                azure_endpoint=azure_endpoint, azure_deployment=azure_deployment, api_version=api_version
-            )
-            # PromptFlow will call DefaultAzureCredential behind the scenes
-        openai_config.model = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"]
+            if tenant_id := os.getenv("AZURE_TENANT_ID"):
+                logger.info("Authenticating to Azure using Azure Developer CLI Credential for tenant %s", tenant_id)
+                azure_credential = azure.identity.AzureDeveloperCliCredential(tenant_id=tenant_id, process_timeout=60)
+            else:
+                logger.info("Authenticating to Azure using Azure Developer CLI Credential")
+                azure_credential = azure.identity.AzureDeveloperCliCredential(process_timeout=60)
+            openai_config: AzureOpenAIModelConfiguration = {
+                "azure_endpoint": azure_endpoint,
+                "azure_deployment": azure_deployment,
+                "credential": azure_credential,
+            }
+            # azure-ai-evaluate will call DefaultAzureCredential behind the scenes,
+            # so we must be logged in to Azure CLI with the correct tenant
+        openai_config["model"] = os.environ["AZURE_OPENAI_EVAL_MODEL"]
     else:
         logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
-        openai_config = OpenAIModelConfiguration(
-            model=os.environ["OPENAICOM_CHAT_MODEL"], api_key=os.environ.get("OPENAICOM_KEY")
-        )
+        openai_config: OpenAIModelConfiguration = {"api_key": os.environ["OPENAICOM_KEY"], "model": "gpt-4"}
     return openai_config
 
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.WARNING)
-    logger.setLevel(logging.INFO)
+    logging.basicConfig(
+        level=logging.INFO, format="%(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)]
+    )
     load_dotenv(".env", override=True)
 
     openai_config = get_openai_config()
     run_evaluate_from_config(
-        working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=20
+        working_dir=Path(__file__).parent, config_path="eval_config.json", openai_config=openai_config, num_questions=2
     )

evals/generate_ground_truth.py

@@ -56,7 +56,7 @@ def get_openai_config_dict() -> dict:
             api_key = os.environ["AZURE_OPENAI_KEY"]
         else:
             logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential")
-            azure_credential = DefaultAzureCredential()
+            azure_credential = DefaultAzureCredential(process_timeout=60)
             api_key = azure_credential.get_token("https://cognitiveservices.azure.com/.default").token
         openai_config = {
             "api_type": "azure",