From 07a81d70be67eaee43d13ff8818ffdaa825f1026 Mon Sep 17 00:00:00 2001
From: Xubin Ren <52506698+Re-bin@users.noreply.github.com>
Date: Fri, 24 Jul 2026 14:55:38 +0800
Subject: [PATCH] fix(providers): preserve multimodal tool outputs

---
 .../providers/openai_responses/converters.py  | 47 ++++++++++++++++++-
 tests/providers/test_openai_responses.py      | 43 +++++++++++++++++
 2 files changed, 88 insertions(+), 2 deletions(-)

diff --git a/nanobot/providers/openai_responses/converters.py b/nanobot/providers/openai_responses/converters.py
index c8b756b1..b9149d51 100644
--- a/nanobot/providers/openai_responses/converters.py
+++ b/nanobot/providers/openai_responses/converters.py
@@ -54,8 +54,8 @@ def convert_messages(messages: list[dict[str, Any]]) -> tuple[str, list[dict[str
 
         if role == "tool":
             call_id, _ = split_tool_call_id(msg.get("tool_call_id"))
-            output_text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
-            input_items.append({"type": "function_call_output", "call_id": call_id, "output": output_text})
+            output = convert_tool_output(content)
+            input_items.append({"type": "function_call_output", "call_id": call_id, "output": output})
 
     return system_prompt, input_items
 
@@ -84,6 +84,49 @@ def convert_user_message(content: Any) -> dict[str, Any]:
     return {"role": "user", "content": [{"type": "input_text", "text": ""}]}
 
 
+def convert_tool_output(content: Any) -> str | list[dict[str, Any]]:
+    """Convert a tool result to Responses API function-call output content.
+
+    The Responses API accepts text, image, and file blocks as function tool
+    output. Nanobot's file tools use Chat Completions-style ``text`` and
+    ``image_url`` blocks for image reads; serializing those blocks as JSON
+    turns the image into inert text and can make the request unnecessarily
+    large. Preserve supported multimodal blocks and strip internal metadata.
+    """
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        converted: list[dict[str, Any]] = []
+        for item in content:
+            if not isinstance(item, dict):
+                continue
+            item_type = item.get("type")
+            if item_type in {"text", "input_text"}:
+                text = item.get("text")
+                if isinstance(text, str):
+                    converted.append({"type": "input_text", "text": text})
+            elif item_type in {"image_url", "input_image"}:
+                image = item.get("image_url")
+                url = image.get("url") if isinstance(image, dict) else image
+                if isinstance(url, str) and url:
+                    converted.append({
+                        "type": "input_image",
+                        "image_url": url,
+                        "detail": item.get("detail", "auto"),
+                    })
+            elif item_type in {"file", "input_file"}:
+                block = {"type": "input_file"}
+                for key in ("file_data", "file_id", "file_url", "filename"):
+                    value = item.get(key)
+                    if isinstance(value, str) and value:
+                        block[key] = value
+                if len(block) > 1:
+                    converted.append(block)
+        if converted:
+            return converted
+    return json.dumps(content, ensure_ascii=False)
+
+
 def convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
     """Convert OpenAI function-calling tool schema to Responses API flat format."""
     converted: list[dict[str, Any]] = []
diff --git a/tests/providers/test_openai_responses.py b/tests/providers/test_openai_responses.py
index 4c2251c0..c36fa098 100644
--- a/tests/providers/test_openai_responses.py
+++ b/tests/providers/test_openai_responses.py
@@ -240,6 +240,49 @@ class TestConvertMessages:
         }])
         assert items[0]["output"] == '{"key": "value"}'
 
+    def test_tool_message_preserves_image_content(self):
+        _, items = convert_messages([{
+            "role": "tool",
+            "tool_call_id": "call_1",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,abc"},
+                    "_meta": {"path": "/private/image.png"},
+                },
+                {"type": "text", "text": "(Image file: image.png)"},
+            ],
+        }])
+
+        assert items[0]["output"] == [
+            {
+                "type": "input_image",
+                "image_url": "data:image/png;base64,abc",
+                "detail": "auto",
+            },
+            {"type": "input_text", "text": "(Image file: image.png)"},
+        ]
+        assert "_meta" not in str(items[0])
+
+    def test_tool_message_preserves_file_content(self):
+        _, items = convert_messages([{
+            "role": "tool",
+            "tool_call_id": "call_1",
+            "content": [{
+                "type": "input_file",
+                "file_id": "file_123",
+                "filename": "report.pdf",
+                "_meta": {"path": "/private/report.pdf"},
+            }],
+        }])
+
+        assert items[0]["output"] == [{
+            "type": "input_file",
+            "file_id": "file_123",
+            "filename": "report.pdf",
+        }]
+        assert "_meta" not in str(items[0])
+
     def test_non_standard_keys_not_leaked(self):
         """Extra keys on messages must not appear in converted items."""
         _, items = convert_messages([{