Add image file viewing support to FileEditor

FileEditor now supports viewing image files (.png, .jpg, .jpeg, .gif, .webp, .bmp) by returning base64-encoded image data and displaying it as ImageContent. Updated FileEditorObservation and tool description to handle image data, and added tests to verify image handling and backward compatibility with text files.

Author: SmartManoj

openhands-tools/openhands/tools/file_editor/definition.py

@@ -85,14 +85,39 @@ class FileEditorObservation(Observation):
         default=None, description="The content of the file after the edit."
     )
     error: str | None = Field(default=None, description="Error message if any.")
+    image_data: str | None = Field(
+        default=None, description="Base64-encoded image data if viewing an image file."
+    )
 
     _diff_cache: Text | None = PrivateAttr(default=None)
 
     @property
     def to_llm_content(self) -> Sequence[TextContent | ImageContent]:
         if self.error:
             return [TextContent(text=self.error)]
-        return [TextContent(text=self.output)]
+        
+        content: list[TextContent | ImageContent] = [TextContent(text=self.output)]
+        
+        # If image_data is present, add it as ImageContent
+        if self.image_data:
+            # Detect MIME type from the data prefix
+            mime_type = "image/png"  # default
+            if self.image_data.startswith("/9j/"):
+                mime_type = "image/jpeg"
+            elif self.image_data.startswith("iVBORw0KGgo"):
+                mime_type = "image/png"
+            elif self.image_data.startswith("R0lGODlh"):
+                mime_type = "image/gif"
+            elif self.image_data.startswith("UklGR"):
+                mime_type = "image/webp"
+            elif self.image_data.startswith("Qk"):
+                mime_type = "image/bmp"
+            
+            # Convert base64 to data URL format for ImageContent
+            data_url = f"data:{mime_type};base64,{self.image_data}"
+            content.append(ImageContent(image_urls=[data_url]))
+        
+        return content
 
     @property
     def visualize(self) -> Text:
@@ -157,6 +182,7 @@ def _has_meaningful_diff(self) -> bool:
 TOOL_DESCRIPTION = """Custom editing tool for viewing, creating and editing files in plain-text format
 * State is persistent across command calls and discussions with the user
 * If `path` is a text file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep
+* If `path` is an image file (.png, .jpg, .jpeg, .gif, .webp, .bmp), `view` displays the image content
 * The `create` command cannot be used if the specified `path` already exists as a file
 * If a `command` generates a long output, it will be truncated and marked with `<response clipped>`
 * The `undo_edit` command will revert the last edit made to the file at `path`

openhands-tools/openhands/tools/file_editor/editor.py

@@ -1,3 +1,4 @@
+import base64
 import os
 import re
 import shutil
@@ -36,6 +37,9 @@
 
 logger = get_logger(__name__)
 
+# Supported image extensions for viewing as base64-encoded content
+IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
+
 
 class FileEditor:
     """
@@ -321,6 +325,28 @@ def view(
                 prev_exist=True,
             )
 
+        # Check if the file is an image
+        file_extension = path.suffix.lower()
+        if file_extension in IMAGE_EXTENSIONS:
+            # Read image file as base64
+            try:
+                with open(path, "rb") as f:
+                    image_bytes = f.read()
+                image_base64 = base64.b64encode(image_bytes).decode("utf-8")
+                
+                output_msg = f"Image file {path} read successfully. Displaying image content."
+                return FileEditorObservation(
+                    command="view",
+                    output=output_msg,
+                    path=str(path),
+                    prev_exist=True,
+                    image_data=image_base64,
+                )
+            except Exception as e:
+                raise ToolError(
+                    f"Failed to read image file {path}: {e}"
+                ) from None
+
         # Validate file and count lines
         self.validate_file(path)
         num_lines = self._count_lines(path)
@@ -603,8 +629,9 @@ def validate_file(self, path: Path) -> None:
                 ),
             )
 
-        # Check file type
-        if is_binary(str(path)):
+        # Check file type - allow image files
+        file_extension = path.suffix.lower()
+        if is_binary(str(path)) and file_extension not in IMAGE_EXTENSIONS:
             raise FileValidationError(
                 path=str(path),
                 reason=(