fix(vertexai): ChatAnthropicVertex support images with langchain 1.X

fixes #1380

Author: qvalentin

libs/vertexai/langchain_google_vertexai/_anthropic_utils.py

@@ -187,39 +187,40 @@ def _format_message_anthropic(
                         new_block[copy_attr] = block[copy_attr]
 
                 if block["type"] == "image":
-                    if block["source_type"] == "url":
-                        if block["url"].startswith("data:"):
+                    if "url" in block:
+                        url = block["url"]
+                        if url.startswith("data:"):
                             # Data URI
                             formatted_block = {
                                 "type": "image",
-                                "source": _format_image(block["url"], project),
+                                "source": _format_image(url, project),
                             }
                         else:
                             formatted_block = {
                                 "type": "image",
-                                "source": {"type": "url", "url": block["url"]},
+                                "source": {"type": "url", "url": url},
                             }
-                    elif block["source_type"] == "base64":
+                    elif "base64" in block:
                         formatted_block = {
                             "type": "image",
                             "source": {
                                 "type": "base64",
                                 "media_type": block["mime_type"],
-                                "data": block["data"],
+                                "data": block["base64"],
                             },
                         }
-                    elif block["source_type"] == "id":
+                    elif "file_id" in block:
                         formatted_block = {
                             "type": "image",
                             "source": {
                                 "type": "file",
-                                "file_id": block["id"],
+                                "file_id": block["file_id"],
                             },
                         }
                     else:
                         msg = (
-                            "Anthropic only supports 'url' and 'base64' source_type "
-                            "for image content blocks."
+                            "Image content blocks must have either 'url', 'base64', "
+                            "or 'file_id' field."
                         )
                         raise ValueError(msg)
                     content.append(formatted_block)

libs/vertexai/tests/unit_tests/test_anthropic_utils.py

@@ -2,6 +2,7 @@
 
 import base64
 from unittest.mock import patch
+
 import pytest
 from anthropic.types import (
     RawContentBlockDeltaEvent,
@@ -15,18 +16,18 @@
     SystemMessage,
     ToolMessage,
 )
+from langchain_core.messages.content import create_image_block, create_text_block
 from langchain_core.messages.tool import tool_call as create_tool_call
 
 from langchain_google_vertexai._anthropic_utils import (
     _documents_in_params,
+    _format_image,
     _format_message_anthropic,
     _format_messages_anthropic,
     _make_message_chunk_from_anthropic_event,
     _thinking_in_params,
 )
 
-from langchain_google_vertexai._anthropic_utils import _format_image
-
 
 def test_format_message_anthropic_with_cache_control_in_kwargs() -> None:
     """Test formatting a message with cache control in additional_kwargs."""
@@ -785,25 +786,11 @@ def test_format_messages_anthropic_with_mixed_messages() -> None:
         (
             [
                 AIMessage(
-                    content=[
-                        {"type": "text", "text": "Text content"},
-                        {
-                            "type": "image",
-                            "source_type": "url",
-                            "url": "https://example.com/image.png",
-                        },
-                        {
-                            "type": "image",
-                            "source_type": "url",
-                            "url": "data:image/png;base64,/9j/4AAQSk",
-                        },
-                        {
-                            "type": "image",
-                            "source_type": "base64",
-                            "mime_type": "image/png",
-                            "data": "/9j/4AAQSk",
-                        },
-                        {"type": "image", "source_type": "id", "id": "1"},
+                    content_blocks=[
+                        create_text_block(text="Text content"),
+                        create_image_block(url="https://example.com/image.png"),
+                        create_image_block(base64="/9j/4AAQSk", mime_type="image/png"),
+                        create_image_block(file_id="1"),
                     ]
                 ),
             ],
@@ -828,14 +815,6 @@ def test_format_messages_anthropic_with_mixed_messages() -> None:
                                 "data": "/9j/4AAQSk",
                             },
                         },
-                        {
-                            "type": "image",
-                            "source": {
-                                "type": "base64",
-                                "media_type": "image/png",
-                                "data": "/9j/4AAQSk",
-                            },
-                        },
                         {"type": "image", "source": {"type": "file", "file_id": "1"}},
                     ],
                 }