fix(providers): preserve multimodal tool outputs

This commit is contained in:
Xubin Ren
2026-07-24 18:58:32 +08:00
parent d3e4b35f2b
commit 07a81d70be
2 changed files with 88 additions and 2 deletions
@@ -54,8 +54,8 @@ def convert_messages(messages: list[dict[str, Any]]) -> tuple[str, list[dict[str
if role == "tool":
call_id, _ = split_tool_call_id(msg.get("tool_call_id"))
output_text = content if isinstance(content, str) else json.dumps(content, ensure_ascii=False)
input_items.append({"type": "function_call_output", "call_id": call_id, "output": output_text})
output = convert_tool_output(content)
input_items.append({"type": "function_call_output", "call_id": call_id, "output": output})
return system_prompt, input_items
@@ -84,6 +84,49 @@ def convert_user_message(content: Any) -> dict[str, Any]:
return {"role": "user", "content": [{"type": "input_text", "text": ""}]}
def convert_tool_output(content: Any) -> str | list[dict[str, Any]]:
"""Convert a tool result to Responses API function-call output content.
The Responses API accepts text, image, and file blocks as function tool
output. Nanobot's file tools use Chat Completions-style ``text`` and
``image_url`` blocks for image reads; serializing those blocks as JSON
turns the image into inert text and can make the request unnecessarily
large. Preserve supported multimodal blocks and strip internal metadata.
"""
if isinstance(content, str):
return content
if isinstance(content, list):
converted: list[dict[str, Any]] = []
for item in content:
if not isinstance(item, dict):
continue
item_type = item.get("type")
if item_type in {"text", "input_text"}:
text = item.get("text")
if isinstance(text, str):
converted.append({"type": "input_text", "text": text})
elif item_type in {"image_url", "input_image"}:
image = item.get("image_url")
url = image.get("url") if isinstance(image, dict) else image
if isinstance(url, str) and url:
converted.append({
"type": "input_image",
"image_url": url,
"detail": item.get("detail", "auto"),
})
elif item_type in {"file", "input_file"}:
block = {"type": "input_file"}
for key in ("file_data", "file_id", "file_url", "filename"):
value = item.get(key)
if isinstance(value, str) and value:
block[key] = value
if len(block) > 1:
converted.append(block)
if converted:
return converted
return json.dumps(content, ensure_ascii=False)
def convert_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Convert OpenAI function-calling tool schema to Responses API flat format."""
converted: list[dict[str, Any]] = []