set default temperature to 0 in generation config (#814)

* set default temperature to 0 in generation config

* issue warning when temperature == 0 with multiple samples

* fix test

Author: NathanHB

src/lighteval/models/litellm_model.py

@@ -114,7 +114,7 @@ def _prepare_max_new_tokens(self, max_new_tokens):
             max_new_tokens = min(max_new_tokens * 10, 32000)
         return max_new_tokens
 
-    def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_sequence):
+    def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_sequence):  # noqa: C901
         """Make API call with retries."""
         response = ModelResponse()
         for attempt in range(self.API_MAX_RETRY):
@@ -135,6 +135,12 @@ def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_se
                     "caching": True,
                     "api_key": self.api_key,
                 }
+
+                if num_samples > 1 and self.generation_parameters.temperature == 0:
+                    logger.warning(
+                        "num_samples > 1 but temperature is set to 0, this will not sample different outputs."
+                    )
+
                 if "o1" in self.model:
                     logger.warning("O1 models do not support temperature, top_p, stop sequence. Disabling.")
                 else:

src/lighteval/models/model_input.py

@@ -36,7 +36,9 @@ class GenerationParameters(BaseModel, extra="forbid"):
 
     seed: NonNegativeInt | None = None  # vllm, tgi, litellm
     stop_tokens: list[str] | None = None  # vllm, transformers, tgi, litellm, sglang
-    temperature: NonNegativeFloat | None = None  # vllm, transformers, tgi, litellm, sglang
+    temperature: NonNegativeFloat = (
+        0  # vllm, transformers, tgi, litellm, sglang # if not set, defaults to greedy decoding
+    )
     top_k: NonNegativeInt | None = None  # vllm, transformers, tgi, sglang
     min_p: NonNegativeFloat | None = None  # vllm, transformers, sglang
     top_p: NonNegativeFloat | None = None  # vllm, transformers, tgi, litellm, sglang

src/lighteval/models/sglang/sglang_model.py

@@ -258,6 +258,8 @@ def _generate(
             self.sampling_params["max_new_tokens"] = max_new_tokens
             self.sampling_params["stop"] = stop_tokens
             self.sampling_params["n"] = num_samples
+            if num_samples > 1 and self.sampling_params["temperature"] == 0:
+                logger.warning("num_samples > 1 but temperature is set to 0, this will not sample different outputs.")
         else:
             self.sampling_params["max_new_tokens"] = 1
             self.sampling_params["temperature"] = 0

src/lighteval/models/transformers/transformers_model.py

@@ -636,7 +636,7 @@ def _generate(
         max_new_tokens: int,
         stop_tokens: list[str],
         returns_logits: Optional[bool] = False,
-        num_samples: Optional[int] = 1,
+        num_samples: int = 1,
         do_sample: Optional[bool] = False,
     ) -> list[GenerativeResponse]:
         """Contains the actual logic of the generation.
@@ -655,6 +655,8 @@ def _generate(
             output_logits=returns_logits,
             renormalize_logits=True,
         )
+        if num_samples > 1 and generation_config["temperature"] == 0:
+            logger.warning("num_samples > 1 but temperature is set to 0, this will not sample different outputs.")
 
         # Compute model generation
         outputs: GenerateOutput = self.model.generate(

src/lighteval/models/vllm/vllm_model.py

@@ -336,6 +336,8 @@ def _generate(
             sampling_params.stop = stop_tokens
             sampling_params.logprobs = 1 if returns_logits else 0
 
+            if num_samples > 1 and sampling_params.temperature == 0:
+                logger.warning("num_samples > 1 but temperature is set to 0, this will not sample different outputs.")
         else:
             sampling_params.temperature = 0
             sampling_params.prompt_logprobs = 1

tests/models/endpoints/test_tgi_model.py

@@ -47,7 +47,7 @@ class TestTGIModelConfig:
                         "repetition_penalty": None,
                         "seed": None,
                         "stop_tokens": None,
-                        "temperature": None,
+                        "temperature": 0,
                         "top_k": None,
                         "top_p": None,
                         "truncate_prompt": None,