From ae46b4390c940dc56b848363a635dcbfb5bbbd04 Mon Sep 17 00:00:00 2001 From: "rok.kozel" Date: Thu, 11 Sep 2025 18:42:34 +0200 Subject: [PATCH] fix: make top_p parameter optional for Azure AI models Azure OpenAI models don't support the top_p parameter, which was causing API errors. This change adds conditional logic to only include top_p in model_kwargs when it's defined in the model configuration. Tested with Azure AI Foundry. - Add conditional checks for top_p in simple_chat.py and websocket_wiki.py - Azure models in generator.json already omit top_p (only have temperature) - Other providers continue to use their configured top_p values - Backwards compatible with existing configurations --- api/simple_chat.py | 6 ++++-- api/websocket_wiki.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/simple_chat.py b/api/simple_chat.py index 06d329a28..dae0625be 100644 --- a/api/simple_chat.py +++ b/api/simple_chat.py @@ -423,9 +423,11 @@ async def chat_completions_stream(request: ChatCompletionRequest): model_kwargs = { "model": request.model, "stream": True, - "temperature": model_config["temperature"], - "top_p": model_config["top_p"] + "temperature": model_config["temperature"] } + # Only add top_p if it exists in the model config + if "top_p" in model_config: + model_kwargs["top_p"] = model_config["top_p"] api_kwargs = model.convert_inputs_to_api_kwargs( input=prompt, diff --git a/api/websocket_wiki.py b/api/websocket_wiki.py index 2a7cce9e3..93e7ebf40 100644 --- a/api/websocket_wiki.py +++ b/api/websocket_wiki.py @@ -502,9 +502,11 @@ async def handle_websocket_chat(websocket: WebSocket): model_kwargs = { "model": request.model, "stream": True, - "temperature": model_config["temperature"], - "top_p": model_config["top_p"] + "temperature": model_config["temperature"] } + # Only add top_p if it exists in the model config + if "top_p" in model_config: + model_kwargs["top_p"] = model_config["top_p"] api_kwargs = model.convert_inputs_to_api_kwargs( input=prompt,