Use SDK for stream
This commit is contained in:
Kunal Karmakar
committed by
Xubin Ren
Xubin Ren
parent
0417c3f03b
commit
8c0607e079
@@ -11,12 +11,11 @@ import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from nanobot.providers.base import LLMProvider, LLMResponse
|
||||
from nanobot.providers.openai_responses_common import (
|
||||
consume_sse,
|
||||
consume_sdk_stream,
|
||||
convert_messages,
|
||||
convert_tools,
|
||||
parse_response_output,
|
||||
@@ -94,6 +93,7 @@ class AzureOpenAIProvider(LLMProvider):
|
||||
"model": deployment,
|
||||
"instructions": instructions or None,
|
||||
"input": input_items,
|
||||
"max_output_tokens": max(1, max_tokens),
|
||||
"store": False,
|
||||
"stream": False,
|
||||
}
|
||||
@@ -159,31 +159,15 @@ class AzureOpenAIProvider(LLMProvider):
|
||||
body["stream"] = True
|
||||
|
||||
try:
|
||||
# Use raw httpx stream via the SDK's base URL so we can reuse
|
||||
# the shared Responses-API SSE parser (same as Codex provider).
|
||||
base_url = str(self._client.base_url).rstrip("/")
|
||||
url = f"{base_url}/responses"
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._client.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
**(self._client._custom_headers or {}),
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=60.0, verify=True) as http:
|
||||
async with http.stream("POST", url, headers=headers, json=body) as response:
|
||||
if response.status_code != 200:
|
||||
text = await response.aread()
|
||||
return LLMResponse(
|
||||
content=f"Azure OpenAI API Error {response.status_code}: {text.decode('utf-8', 'ignore')}",
|
||||
finish_reason="error",
|
||||
)
|
||||
content, tool_calls, finish_reason = await consume_sse(
|
||||
response, on_content_delta,
|
||||
)
|
||||
return LLMResponse(
|
||||
content=content or None,
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
stream = await self._client.responses.create(**body)
|
||||
content, tool_calls, finish_reason = await consume_sdk_stream(
|
||||
stream, on_content_delta,
|
||||
)
|
||||
return LLMResponse(
|
||||
content=content or None,
|
||||
tool_calls=tool_calls,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
except Exception as e:
|
||||
return self._handle_error(e)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from nanobot.providers.openai_responses_common.converters import (
|
||||
)
|
||||
from nanobot.providers.openai_responses_common.parsing import (
|
||||
FINISH_REASON_MAP,
|
||||
consume_sdk_stream,
|
||||
consume_sse,
|
||||
iter_sse,
|
||||
map_finish_reason,
|
||||
@@ -21,6 +22,7 @@ __all__ = [
|
||||
"split_tool_call_id",
|
||||
"iter_sse",
|
||||
"consume_sse",
|
||||
"consume_sdk_stream",
|
||||
"map_finish_reason",
|
||||
"parse_response_output",
|
||||
"FINISH_REASON_MAP",
|
||||
|
||||
@@ -171,3 +171,72 @@ def parse_response_output(response: Any) -> LLMResponse:
|
||||
finish_reason=finish_reason,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
|
||||
async def consume_sdk_stream(
|
||||
stream: Any,
|
||||
on_content_delta: Callable[[str], Awaitable[None]] | None = None,
|
||||
) -> tuple[str, list[ToolCallRequest], str]:
|
||||
"""Consume an SDK async stream from ``client.responses.create(stream=True)``.
|
||||
|
||||
The SDK yields typed event objects with a ``.type`` attribute and
|
||||
event-specific fields. Returns ``(content, tool_calls, finish_reason)``.
|
||||
"""
|
||||
content = ""
|
||||
tool_calls: list[ToolCallRequest] = []
|
||||
tool_call_buffers: dict[str, dict[str, Any]] = {}
|
||||
finish_reason = "stop"
|
||||
|
||||
async for event in stream:
|
||||
event_type = getattr(event, "type", None)
|
||||
if event_type == "response.output_item.added":
|
||||
item = getattr(event, "item", None)
|
||||
if item and getattr(item, "type", None) == "function_call":
|
||||
call_id = getattr(item, "call_id", None)
|
||||
if not call_id:
|
||||
continue
|
||||
tool_call_buffers[call_id] = {
|
||||
"id": getattr(item, "id", None) or "fc_0",
|
||||
"name": getattr(item, "name", None),
|
||||
"arguments": getattr(item, "arguments", None) or "",
|
||||
}
|
||||
elif event_type == "response.output_text.delta":
|
||||
delta_text = getattr(event, "delta", "") or ""
|
||||
content += delta_text
|
||||
if on_content_delta and delta_text:
|
||||
await on_content_delta(delta_text)
|
||||
elif event_type == "response.function_call_arguments.delta":
|
||||
call_id = getattr(event, "call_id", None)
|
||||
if call_id and call_id in tool_call_buffers:
|
||||
tool_call_buffers[call_id]["arguments"] += getattr(event, "delta", "") or ""
|
||||
elif event_type == "response.function_call_arguments.done":
|
||||
call_id = getattr(event, "call_id", None)
|
||||
if call_id and call_id in tool_call_buffers:
|
||||
tool_call_buffers[call_id]["arguments"] = getattr(event, "arguments", "") or ""
|
||||
elif event_type == "response.output_item.done":
|
||||
item = getattr(event, "item", None)
|
||||
if item and getattr(item, "type", None) == "function_call":
|
||||
call_id = getattr(item, "call_id", None)
|
||||
if not call_id:
|
||||
continue
|
||||
buf = tool_call_buffers.get(call_id) or {}
|
||||
args_raw = buf.get("arguments") or getattr(item, "arguments", None) or "{}"
|
||||
try:
|
||||
args = json.loads(args_raw)
|
||||
except Exception:
|
||||
args = {"raw": args_raw}
|
||||
tool_calls.append(
|
||||
ToolCallRequest(
|
||||
id=f"{call_id}|{buf.get('id') or getattr(item, 'id', None) or 'fc_0'}",
|
||||
name=buf.get("name") or getattr(item, "name", None),
|
||||
arguments=args,
|
||||
)
|
||||
)
|
||||
elif event_type == "response.completed":
|
||||
resp = getattr(event, "response", None)
|
||||
status = getattr(resp, "status", None) if resp else None
|
||||
finish_reason = map_finish_reason(status)
|
||||
elif event_type in {"error", "response.failed"}:
|
||||
raise RuntimeError("Response failed")
|
||||
|
||||
return content, tool_calls, finish_reason
|
||||
|
||||
Reference in New Issue
Block a user