Update to Torch 2.7.0 (#3221)

* Update to Torch 2.7.0

* Try to fix typer/click issue

* Pin click to fix incompatibility with typer

* Fix some test outputs with slight deviations

* Attempt again to sync with CI

* Mamba too

* Fixup mllama

Also switch to `unsloth/Llama-3.2-11B-Vision-Instruct` for testing
from the EU :).

Author: danieldk

Dockerfile

@@ -48,7 +48,7 @@ FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS pytorch-install
 WORKDIR /usr/src/
 
 # NOTE: When updating PyTorch version, beware to remove `pip install nvidia-nccl-cu12==2.22.3` below in the Dockerfile. Context: https://github.com/huggingface/text-generation-inference/pull/2099
-ARG PYTORCH_VERSION=2.6
+ARG PYTORCH_VERSION=2.7
 ARG PYTHON_VERSION=3.11
 
 # Keep in sync with `server/pyproject.toml

flake.lock

@@ -5,7 +5,7 @@
       inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
     };
     nix-filter.url = "github:numtide/nix-filter";
-    tgi-nix.url = "github:huggingface/text-generation-inference-nix";
+    tgi-nix.url = "github:huggingface/text-generation-inference-nix/torch-2.7";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
     rust-overlay = {

flake.nix

@@ -5,22 +5,22 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "Okay, let's analyze the image.\n\nThe image is a solid, bright white color. There is nothing else visible within it. \n\nIt's essentially a blank white canvas or a completely white square. \n\nIs there anything specific you'd like me to do with this image, such as describe it further or imagine what it might represent?",
+        "content": "Okay, let's analyze the image.\n\nThe image is a solid, bright white color. There is nothing else visible within it. \n\nIt's essentially a blank white square or rectangle.",
         "name": null,
         "role": "assistant",
         "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1741965894,
+  "created": 1747062956,
   "id": "",
   "model": "google/gemma-3-4b-it",
   "object": "chat.completion",
-  "system_fingerprint": "3.2.1-dev0-native",
+  "system_fingerprint": "3.3.0-dev0-native",
   "usage": {
-    "completion_tokens": 74,
+    "completion_tokens": 42,
     "prompt_tokens": 277,
-    "total_tokens": 351
+    "total_tokens": 319
   }
 }

integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_jpg.json

@@ -1,26 +1,26 @@
 {
   "choices": [
     {
-      "finish_reason": "length",
+      "finish_reason": "stop",
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "Okay, let's analyze the image. \n\nThe image is entirely white, with a very subtle, faint outline of a stylized, cartoonish figure. It appears to be a simplified depiction of a person, likely a child, with a wide-eyed expression and a small, rounded body. \n\nIt's almost like a minimalist, iconic representation. \n\nDo you want me to try and describe it in more detail, or perhaps suggest what this image might represent (e.g",
+        "content": "Okay, let's analyze the image. \n\nThe image is a very plain, solid white square. That's it! \n\nIt's essentially a blank canvas. \n\nDo you want me to describe it in more detail, or are you interested in something else regarding this image?",
         "name": null,
         "role": "assistant",
         "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1744396706,
+  "created": 1747062955,
   "id": "",
   "model": "google/gemma-3-4b-it",
   "object": "chat.completion",
   "system_fingerprint": "3.3.0-dev0-native",
   "usage": {
-    "completion_tokens": 100,
+    "completion_tokens": 62,
     "prompt_tokens": 277,
-    "total_tokens": 377
+    "total_tokens": 339
   }
 }

integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgb_png.json

@@ -5,22 +5,22 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "Okay, let's analyze the image. \n\nThe transparent image reveals a stylized depiction of **a human head**. It's a minimalist, geometric representation, showing the basic shapes of the skull, eye sockets, and head outline. \n\nIf you'd like, you can give me more details about the image or ask me to focus on a specific aspect of it.",
+        "content": "Okay, let's analyze the image. \n\nThe transparent image reveals a stylized depiction of **a human head**. It's a minimalist, geometric representation, showing the basic shapes of the skull, eye sockets, and head outline. \n\nDo you want me to describe any specific element of the image in more detail?",
         "name": null,
         "role": "assistant",
         "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1744396703,
+  "created": 1747062952,
   "id": "",
   "model": "google/gemma-3-4b-it",
   "object": "chat.completion",
   "system_fingerprint": "3.3.0-dev0-native",
   "usage": {
-    "completion_tokens": 78,
+    "completion_tokens": 67,
     "prompt_tokens": 277,
-    "total_tokens": 355
+    "total_tokens": 344
   }
 }

integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_base64_rgba.json

@@ -5,22 +5,22 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The beach has turquoise water and a distant island visible in the background. The sky is bright blue with some white clouds. \n\nIt's a quite a humorous and unusual scene – a cow enjoying a day at the beach!",
+        "content": "Here's a description of what's shown in the image:\n\nThe image depicts a brown cow standing on a sandy beach. The beach has turquoise water and a distant island visible in the background. The sky is bright blue with some white clouds. \n\nIt's a quite a humorous and unusual scene – a cow enjoying a beach day!",
         "name": null,
         "role": "assistant",
         "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1744396699,
+  "created": 1747216083,
   "id": "",
   "model": "google/gemma-3-4b-it",
   "object": "chat.completion",
   "system_fingerprint": "3.3.0-dev0-native",
   "usage": {
-    "completion_tokens": 74,
+    "completion_tokens": 72,
     "prompt_tokens": 275,
-    "total_tokens": 349
+    "total_tokens": 347
   }
 }

integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow.json

@@ -5,22 +5,22 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "That's a fantastic question! However, the image doesn't show a dog. It shows a **Brown Swiss cow** standing on a beach. \n\nBrown Swiss cows are known for their reddish-brown color and distinctive white markings. \n\nIf you'd like, you can send me another image and I’ll do my best to identify it!",
+        "content": "That's a fantastic question! However, the image doesn't show a dog. It shows a **Brown Swiss cow** standing on a beach. \n\nBrown Swiss cows are known for their beautiful reddish-brown coats and distinctive white markings. \n\nIf you'd like, you can send me another image, and I'll do my best to identify the animal in it!",
         "name": null,
         "role": "assistant",
         "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1744396697,
+  "created": 1747216080,
   "id": "",
   "model": "google/gemma-3-4b-it",
   "object": "chat.completion",
   "system_fingerprint": "3.3.0-dev0-native",
   "usage": {
-    "completion_tokens": 75,
+    "completion_tokens": 80,
     "prompt_tokens": 279,
-    "total_tokens": 354
+    "total_tokens": 359
   }
 }

integration-tests/models/__snapshots__/test_flash_gemma3/test_flash_gemma3_image_cow_dog.json

@@ -8,126 +8,126 @@
     "tokens": [
       {
         "id": 108,
-        "logprob": -0.73046875,
+        "logprob": -0.48046875,
         "special": false,
         "text": "\n"
       },
       {
         "id": 30234,
-        "logprob": -2.328125,
+        "logprob": -2.21875,
         "special": false,
         "text": "Brown"
       },
       {
         "id": 108,
-        "logprob": -0.12060547,
+        "logprob": -0.119140625,
         "special": false,
         "text": "\n"
       },
       {
         "id": 3726,
-        "logprob": -1.7734375,
+        "logprob": -1.703125,
         "special": false,
         "text": "Car"
       },
       {
         "id": 108,
-        "logprob": -0.041503906,
+        "logprob": -0.0390625,
         "special": false,
         "text": "\n"
       },
       {
         "id": 2915,
-        "logprob": -1.796875,
+        "logprob": -1.8203125,
         "special": false,
         "text": "Color"
       },
       {
         "id": 108,
-        "logprob": -0.039794922,
+        "logprob": -0.035888672,
         "special": false,
         "text": "\n"
       },
       {
         "id": 19178,
-        "logprob": -1.96875,
+        "logprob": -2.015625,
         "special": false,
         "text": "Cool"
       },
       {
         "id": 108,
-        "logprob": -0.080566406,
+        "logprob": -0.08105469,
         "special": false,
         "text": "\n"
       },
       {
         "id": 40544,
-        "logprob": -2.1875,
+        "logprob": -2.09375,
         "special": false,
         "text": "Decor"
       },
       {
         "id": 108,
-        "logprob": -0.033935547,
+        "logprob": -0.038330078,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 13936,
-        "logprob": -1.6328125,
+        "id": 108,
+        "logprob": -1.515625,
         "special": false,
-        "text": "Green"
+        "text": "\n"
       },
       {
         "id": 108,
-        "logprob": -0.16210938,
+        "logprob": -1.8671875,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 955,
-        "logprob": -2.015625,
+        "id": 108,
+        "logprob": -1.6328125,
         "special": false,
-        "text": "..."
+        "text": "\n"
       },
       {
         "id": 108,
-        "logprob": -0.14746094,
+        "logprob": -1.265625,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 955,
-        "logprob": -0.73828125,
+        "id": 108,
+        "logprob": -1.0078125,
         "special": false,
-        "text": "..."
+        "text": "\n"
       },
       {
         "id": 108,
-        "logprob": -0.051513672,
+        "logprob": -1.03125,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 955,
-        "logprob": -0.34765625,
+        "id": 235336,
+        "logprob": -1.2109375,
         "special": false,
-        "text": "..."
+        "text": "?"
       },
       {
         "id": 108,
-        "logprob": -0.020141602,
+        "logprob": -0.29101562,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 955,
-        "logprob": -0.11767578,
+        "id": 235336,
+        "logprob": -0.08935547,
         "special": false,
-        "text": "..."
+        "text": "?"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\nBrown\nCar\nColor\nCool\nDecor\nGreen\n...\n...\n...\n..."
+  "generated_text": "\nBrown\nCar\nColor\nCool\nDecor\n\n\n\n\n\n\n?\n?"
 }