Handle InternalServerError with choices=None to enable temperature bumping

Some LLM providers (e.g., Gemini) occasionally return malformed responses
with choices=None. When this happens, litellm may raise InternalServerError
during internal validation before returning the response to our code.

Previously, these InternalServerErrors were retried, but without temperature
bumping. Since temperature=0.0 is deterministic, all retries would hit the
same provider bug and fail.

This change detects InternalServerErrors related to malformed choices and
converts them to LLMNoResponseError. This enables the RetryMixin's adaptive
behavior, which bumps temperature from 0.0 to 1.0 on retry. The temperature
change makes the provider's response generation non-deterministic, which can
help avoid the same bug on subsequent attempts.

This complements PR #1107, which handles the case where litellm successfully
returns a response with empty choices. Together, these changes ensure
temperature bumping happens regardless of where the choices validation fails.

Co-authored-by: openhands <openhands@all-hands.dev>

Author: openhands-agent

openhands-sdk/openhands/sdk/llm/llm.py

@@ -518,7 +518,26 @@ def _one_attempt(**retry_kwargs) -> ModelResponse:
             self._telemetry.on_request(log_ctx=log_ctx)
             # Merge retry-modified kwargs (like temperature) with call_kwargs
             final_kwargs = {**call_kwargs, **retry_kwargs}
-            resp = self._transport_call(messages=formatted_messages, **final_kwargs)
+            try:
+                resp = self._transport_call(messages=formatted_messages, **final_kwargs)
+            except InternalServerError as e:
+                # litellm sometimes raises InternalServerError when it receives
+                # a malformed response from the provider (e.g., choices=None).
+                # In these cases, the error typically contains "choices" and
+                # validation-related keywords. We convert these to
+                # LLMNoResponseError so that temperature bumping is triggered on
+                # retry, which can help avoid the same provider bug.
+                error_msg = str(e).lower()
+                if "choices" in error_msg and (
+                    "none" in error_msg
+                    or "assert" in error_msg
+                    or "invalid" in error_msg
+                ):
+                    raise LLMNoResponseError(
+                        f"Provider returned malformed response: {e}"
+                    ) from e
+                raise
+
             raw_resp: ModelResponse | None = None
             if use_mock_tools:
                 raw_resp = copy.deepcopy(resp)

tests/sdk/llm/test_internal_server_error_choices_none.py

@@ -0,0 +1,161 @@
+from unittest.mock import patch
+
+import pytest
+from litellm.exceptions import InternalServerError
+from litellm.types.utils import Choices, Message as LiteLLMMessage, ModelResponse, Usage
+from pydantic import SecretStr
+
+from openhands.sdk.llm import LLM, LLMResponse, Message, TextContent
+from openhands.sdk.llm.exceptions import LLMNoResponseError
+
+
+def create_mock_response(
+    content: str = "ok", response_id: str = "r-1"
+) -> ModelResponse:
+    return ModelResponse(
+        id=response_id,
+        choices=[
+            Choices(
+                finish_reason="stop",
+                index=0,
+                message=LiteLLMMessage(content=content, role="assistant"),
+            )
+        ],
+        created=1,
+        model="gpt-4o",
+        object="chat.completion",
+        system_fingerprint="t",
+        usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
+    )
+
+
+@pytest.fixture
+def base_llm() -> LLM:
+    return LLM(
+        usage_id="test-llm",
+        model="gpt-4o",
+        api_key=SecretStr("test_key"),
+        num_retries=2,
+        retry_min_wait=1,
+        retry_max_wait=2,
+    )
+
+
+@patch("openhands.sdk.llm.llm.litellm_completion")
+def test_internal_server_error_choices_none_retries_then_succeeds(
+    mock_completion, base_llm: LLM
+) -> None:
+    """
+    Test that InternalServerError with choices=None in message is converted
+    to LLMNoResponseError and retried successfully with temperature bump.
+    """
+    mock_completion.side_effect = [
+        InternalServerError(
+            message=(
+                "Invalid response object: assert response_object['choices'] is not None"
+            ),
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+        create_mock_response("success"),
+    ]
+
+    resp = base_llm.completion(
+        messages=[Message(role="user", content=[TextContent(text="hi")])]
+    )
+
+    assert isinstance(resp, LLMResponse)
+    assert resp.message is not None
+    assert mock_completion.call_count == 2
+
+
+@patch("openhands.sdk.llm.llm.litellm_completion")
+def test_internal_server_error_choices_none_exhausts_retries(
+    mock_completion, base_llm: LLM
+) -> None:
+    """
+    Test that when all retries fail with InternalServerError (choices=None),
+    LLMNoResponseError is raised.
+    """
+    mock_completion.side_effect = [
+        InternalServerError(
+            message="Invalid response: choices is None",
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+        InternalServerError(
+            message="Invalid response: choices is None",
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+    ]
+
+    with pytest.raises(LLMNoResponseError) as excinfo:
+        base_llm.completion(
+            messages=[Message(role="user", content=[TextContent(text="hi")])]
+        )
+
+    assert mock_completion.call_count == base_llm.num_retries
+    assert "malformed response" in str(excinfo.value).lower()
+
+
+@patch("openhands.sdk.llm.llm.litellm_completion")
+def test_internal_server_error_choices_none_bumps_temperature(
+    mock_completion, base_llm: LLM
+) -> None:
+    """
+    Test that InternalServerError with choices=None triggers temperature bump.
+    """
+    # Ensure we start at 0.0 to trigger bump to 1.0 on retry
+    assert base_llm.temperature == 0.0
+
+    mock_completion.side_effect = [
+        InternalServerError(
+            message="assert response_object['choices'] is not None",
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+        create_mock_response("ok"),
+    ]
+
+    base_llm.completion(
+        messages=[Message(role="user", content=[TextContent(text="hi")])]
+    )
+
+    # Verify that on the second call, temperature was bumped to 1.0 by RetryMixin
+    assert mock_completion.call_count == 2
+    _, second_kwargs = mock_completion.call_args_list[1]
+    assert second_kwargs.get("temperature") == 1.0
+
+
+@patch("openhands.sdk.llm.llm.litellm_completion")
+def test_internal_server_error_unrelated_not_converted(
+    mock_completion, base_llm: LLM
+) -> None:
+    """
+    Test that unrelated InternalServerError (not about choices) is NOT
+    converted to LLMNoResponseError and is retried as InternalServerError.
+    """
+    mock_completion.side_effect = [
+        InternalServerError(
+            message="Database connection failed",
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+        InternalServerError(
+            message="Database connection failed",
+            llm_provider="test_provider",
+            model="test_model",
+        ),
+    ]
+
+    # Should raise InternalServerError eventually (after mapping to
+    # LLMServiceUnavailableError), not LLMNoResponseError
+    with pytest.raises(Exception) as excinfo:
+        base_llm.completion(
+            messages=[Message(role="user", content=[TextContent(text="hi")])]
+        )
+
+    # Should NOT be LLMNoResponseError
+    assert not isinstance(excinfo.value, LLMNoResponseError)
+    assert mock_completion.call_count == base_llm.num_retries