feat(mcp): deliver image content from MCP tools as artifacts
MCPToolWrapper.execute only handled TextContent; every other block was
rendered with str(block). An MCP ImageContent block therefore became a
large base64 string embedded in the tool result, which (a) was truncated
by max_tool_result_chars, corrupting the data, and (b) could never reach a
channel because it was plain text, not an image artifact.
Decode ImageContent (and EmbeddedResource blobs with an image/* MIME type)
and persist them via store_generated_image_artifact, returning the same
compact {artifacts, next_step} JSON the built-in image_generation tool
produces. The base64 stays out of the model context; the model delivers the
saved file via the message tool's media parameter.
This commit is contained in:
+101
-10
@@ -1,6 +1,7 @@
|
||||
"""MCP client: connects to MCP servers and wraps their tools as native nanobot tools."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
@@ -313,6 +314,52 @@ class _MCPWrapperBase(Tool):
|
||||
return True
|
||||
|
||||
|
||||
def _image_block_data_url(block: Any, types: Any) -> str | None:
|
||||
"""Return a base64 ``data:`` URL for an MCP image-bearing content block.
|
||||
|
||||
Handles ``ImageContent`` directly and ``EmbeddedResource`` wrapping a binary
|
||||
blob with an ``image/*`` MIME type. Returns ``None`` for anything else.
|
||||
``getattr`` guards keep this safe when the installed/faked ``mcp`` SDK does
|
||||
not expose a given type.
|
||||
"""
|
||||
image_cls = getattr(types, "ImageContent", None)
|
||||
if image_cls is not None and isinstance(block, image_cls):
|
||||
mime = getattr(block, "mimeType", None) or "image/png"
|
||||
return f"data:{mime};base64,{block.data}"
|
||||
|
||||
embedded_cls = getattr(types, "EmbeddedResource", None)
|
||||
blob_cls = getattr(types, "BlobResourceContents", None)
|
||||
if embedded_cls is not None and isinstance(block, embedded_cls):
|
||||
resource = getattr(block, "resource", None)
|
||||
if blob_cls is not None and isinstance(resource, blob_cls):
|
||||
mime = getattr(resource, "mimeType", None) or ""
|
||||
if isinstance(mime, str) and mime.startswith("image/"):
|
||||
return f"data:{mime};base64,{resource.blob}"
|
||||
return None
|
||||
|
||||
|
||||
def _mcp_image_tool_result(text_parts: list[str], artifacts: list[dict[str, Any]]) -> str:
|
||||
"""Build the compact tool result for an MCP call that returned image(s).
|
||||
|
||||
The base64 stays out of the model context entirely — only artifact paths and
|
||||
metadata are returned, so the result is small and the channel can deliver the
|
||||
saved file via the message tool.
|
||||
"""
|
||||
payload: dict[str, Any] = {
|
||||
"artifacts": artifacts,
|
||||
"next_step": (
|
||||
"These images were returned by an MCP tool and saved as local artifacts. "
|
||||
"Call the message tool with the artifact 'path' values in the media "
|
||||
"parameter to deliver the images to the user. Do not paste base64 or raw "
|
||||
"paths into your reply unless the user asks for debug details."
|
||||
),
|
||||
}
|
||||
text = "\n".join(part for part in text_parts if part)
|
||||
if text:
|
||||
payload["text"] = text
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
class MCPToolWrapper(_MCPWrapperBase):
|
||||
"""Wraps a single MCP server tool as a nanobot Tool."""
|
||||
|
||||
@@ -340,8 +387,6 @@ class MCPToolWrapper(_MCPWrapperBase):
|
||||
return self._parameters
|
||||
|
||||
async def execute(self, **kwargs: Any) -> str:
|
||||
from mcp import types
|
||||
|
||||
retried_transient = False
|
||||
refreshed_session = False
|
||||
while True:
|
||||
@@ -396,17 +441,63 @@ class MCPToolWrapper(_MCPWrapperBase):
|
||||
)
|
||||
return f"(MCP tool call failed: {type(exc).__name__})"
|
||||
else:
|
||||
# Success — extract result
|
||||
parts = []
|
||||
for block in result.content:
|
||||
if isinstance(block, types.TextContent):
|
||||
parts.append(block.text)
|
||||
else:
|
||||
parts.append(str(block))
|
||||
return "\n".join(parts) or "(no output)"
|
||||
# Success — extract text and persist any image content as artifacts.
|
||||
return self._render_call_result(result.content, kwargs)
|
||||
|
||||
return "(MCP tool call failed)" # Unreachable, but satisfies type checkers
|
||||
|
||||
def _render_call_result(self, content: Any, arguments: Mapping[str, Any]) -> str:
|
||||
"""Turn MCP content blocks into a tool result string.
|
||||
|
||||
Text is concatenated as before. Image blocks are decoded and saved as
|
||||
local artifacts (mirroring the built-in image generation tool) so the
|
||||
model can deliver them via the message tool instead of trying to forward
|
||||
base64 — which would be truncated and bloat the context window.
|
||||
"""
|
||||
from mcp import types
|
||||
|
||||
text_parts: list[str] = []
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
for block in content:
|
||||
if isinstance(block, types.TextContent):
|
||||
text_parts.append(block.text)
|
||||
continue
|
||||
data_url = _image_block_data_url(block, types)
|
||||
if data_url is not None:
|
||||
stored = self._store_image_block(data_url, arguments)
|
||||
if stored is not None:
|
||||
artifacts.append(stored)
|
||||
else:
|
||||
text_parts.append("(MCP tool returned an image that could not be stored)")
|
||||
continue
|
||||
text_parts.append(str(block))
|
||||
|
||||
if artifacts:
|
||||
return _mcp_image_tool_result(text_parts, artifacts)
|
||||
return "\n".join(text_parts) or "(no output)"
|
||||
|
||||
def _store_image_block(
|
||||
self, data_url: str, arguments: Mapping[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
"""Persist one image data URL as an artifact; return its metadata or None."""
|
||||
from nanobot.utils.artifacts import ArtifactError, store_generated_image_artifact
|
||||
|
||||
try:
|
||||
return store_generated_image_artifact(
|
||||
data_url,
|
||||
prompt=str(arguments.get("prompt") or ""),
|
||||
model=str(arguments.get("model") or ""),
|
||||
save_dir="generated",
|
||||
provider=f"mcp:{self._server_name}",
|
||||
)
|
||||
except (ArtifactError, OSError) as exc:
|
||||
logger.warning(
|
||||
"MCP tool '{}' returned an image that could not be stored: {}",
|
||||
self._name,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
class MCPResourceWrapper(_MCPWrapperBase):
|
||||
"""Wraps an MCP resource URI as a read-only nanobot Tool."""
|
||||
|
||||
Reference in New Issue
Block a user