"""Tests for provider progress delta routing in the shared runner.""" import asyncio from unittest.mock import AsyncMock, MagicMock import pytest from agent.runner_helpers import make_run_spec from nanobot.agent.hook import CompositeHook from nanobot.agent.hooks import FileEditActivityHook from nanobot.agent.progress_hook import AgentProgressHook from nanobot.agent.runner import AgentRunner from nanobot.agent.tools.filesystem import EditFileTool, WriteFileTool from nanobot.config.schema import AgentDefaults from nanobot.providers.base import LLMResponse, ToolCallRequest _MAX_TOOL_RESULT_CHARS = AgentDefaults().max_tool_result_chars @pytest.mark.asyncio async def test_runner_can_disable_provider_progress_delta_streaming(): """AgentLoop disables token progress streaming for non-streaming channels.""" provider = MagicMock() provider.supports_progress_deltas = True provider.chat_with_retry = AsyncMock( return_value=LLMResponse(content="done", tool_calls=[], usage={}) ) provider.chat_stream_with_retry = AsyncMock() tools = MagicMock() tools.get_definitions.return_value = [] progress_cb = AsyncMock() runner = AgentRunner() result = await runner.run(make_run_spec(provider, initial_messages=[ {"role": "system", "content": "system"}, {"role": "user", "content": "hi"}, ], tools=tools, model="test-model", max_iterations=1, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, stream_progress_deltas=False, )) assert result.final_content == "done" provider.chat_with_retry.assert_awaited_once() provider.chat_stream_with_retry.assert_not_awaited() progress_cb.assert_not_awaited() @pytest.mark.asyncio async def test_runner_streams_provider_progress_deltas_by_default(): """Direct runner users keep the existing opt-in provider progress behavior.""" provider = MagicMock() provider.supports_progress_deltas = True async def chat_stream_with_retry(*, on_content_delta, **kwargs): await on_content_delta("he") await on_content_delta("llo") return LLMResponse(content="hello", tool_calls=[], usage={}) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = MagicMock() tools.get_definitions.return_value = [] progress_cb = AsyncMock() runner = AgentRunner() result = await runner.run(make_run_spec(provider, initial_messages=[ {"role": "system", "content": "system"}, {"role": "user", "content": "hi"}, ], tools=tools, model="test-model", max_iterations=1, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, )) assert result.final_content == "hello" assert [call.args[0] for call in progress_cb.await_args_list] == ["he", "llo"] provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_routes_hosted_tool_events_to_structured_progress(): provider = MagicMock() provider.supports_progress_deltas = True async def chat_stream_with_retry(*, on_content_delta, on_tool_call_delta, **kwargs): await on_tool_call_delta({ "call_id": "local-call", "name": "read_file", "arguments_delta": "", }) await on_tool_call_delta({ "kind": "hosted_tool", "phase": "start", "call_id": "x-search-1", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": None, }) await on_tool_call_delta({ "kind": "hosted_tool", "phase": "end", "call_id": "x-search-1", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": {"name": "x_semantic_search"}, }) await on_content_delta("done") return LLMResponse(content="done", tool_calls=[], usage={}) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = MagicMock() tools.get_definitions.return_value = [] progress_events: list[dict] = [] progress_text: list[str] = [] async def progress_cb(content, *, tool_events=None, **kwargs): progress_text.append(content) if tool_events: progress_events.extend(tool_events) hook = CompositeHook([AgentProgressHook(on_progress=progress_cb)]) result = await AgentRunner().run(make_run_spec( provider, initial_messages=[{"role": "user", "content": "search X"}], tools=tools, model="test-model", max_iterations=1, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, hook=hook, )) assert result.final_content == "done" assert result.tools_used == [] assert result.tool_events == [] assert progress_events == [ { "version": 1, "phase": "start", "call_id": "x-search-1", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": None, "error": None, "files": [], "embeds": [], }, { "version": 1, "phase": "end", "call_id": "x-search-1", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": {"name": "x_semantic_search"}, "error": None, "files": [], "embeds": [], }, ] assert progress_text == ['search X "nanobot oauth"', "", "done"] provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_fails_pending_hosted_tool_when_model_request_fails(): provider = MagicMock() provider.supports_progress_deltas = True async def chat_stream_with_retry(*, on_tool_call_delta, **kwargs): await on_tool_call_delta({ "kind": "hosted_tool", "phase": "start", "call_id": "x-search-failed", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": None, }) return LLMResponse( content="hosted search backend failed", finish_reason="error", ) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = MagicMock() tools.get_definitions.return_value = [] progress_events: list[dict] = [] async def progress_cb(content, *, tool_events=None, **kwargs): if tool_events: progress_events.extend(tool_events) hook = CompositeHook([AgentProgressHook(on_progress=progress_cb)]) result = await AgentRunner().run(make_run_spec( provider, initial_messages=[{"role": "user", "content": "search X"}], tools=tools, model="test-model", max_iterations=1, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, hook=hook, )) assert result.stop_reason == "error" assert [(event["phase"], event["call_id"]) for event in progress_events] == [ ("start", "x-search-failed"), ("error", "x-search-failed"), ] assert progress_events[-1] == { "version": 1, "phase": "error", "call_id": "x-search-failed", "name": "x_search", "arguments": {"query": "nanobot oauth"}, "result": None, "error": "hosted search backend failed", "files": [], "embeds": [], } provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_emits_write_file_diff_from_tool_execution_snapshots(tmp_path): provider = MagicMock() provider.supports_progress_deltas = True call_count = 0 progress_events: list[dict] = [] (tmp_path / "big.txt").write_text("old\n", encoding="utf-8") async def progress_cb(content, *, file_edit_events=None, **kwargs): if file_edit_events: progress_events.extend(file_edit_events) tool = WriteFileTool(workspace=tmp_path) class Tools: def get_definitions(self): return [{"type": "function", "function": {"name": "write_file"}}] def prepare_call(self, name, params): return tool, params, None async def chat_stream_with_retry(**kwargs): nonlocal call_count call_count += 1 if call_count == 1: return LLMResponse( content=None, tool_calls=[ ToolCallRequest( id="call-write", name="write_file", arguments={"path": "big.txt", "content": "line\n" * 24}, ) ], usage={}, ) return LLMResponse(content="done", tool_calls=[], usage={}) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = Tools() runner = AgentRunner() result = await runner.run(make_run_spec(provider, initial_messages=[{"role": "user", "content": "write a large file"}], tools=tools, model="test-model", max_iterations=2, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, workspace=tmp_path, hook=FileEditActivityHook(on_progress=progress_cb, workspace=tmp_path), )) assert result.final_content == "done" assert progress_events[0]["phase"] == "start" assert progress_events[0]["added"] == 0 assert progress_events[0]["deleted"] == 0 assert any( not event["approximate"] and event["phase"] == "end" and event["added"] == 24 and event["deleted"] == 1 and event["diff"]["format"] == "unified" for event in progress_events ) provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_emits_edit_file_diff_from_tool_execution_snapshots(tmp_path): provider = MagicMock() provider.supports_progress_deltas = True call_count = 0 progress_events: list[dict] = [] target = tmp_path / "notes.txt" target.write_text("old\nkeep\n", encoding="utf-8") async def progress_cb(content, *, file_edit_events=None, **kwargs): if file_edit_events: progress_events.extend(file_edit_events) tool = EditFileTool(workspace=tmp_path) class Tools: def get_definitions(self): return [{"type": "function", "function": {"name": "edit_file"}}] def prepare_call(self, name, params): return tool, params, None async def chat_stream_with_retry(**kwargs): nonlocal call_count call_count += 1 if call_count == 1: return LLMResponse( content=None, tool_calls=[ ToolCallRequest( id="call-edit", name="edit_file", arguments={ "path": "notes.txt", "old_text": "old\nkeep\n", "new_text": "new\nkeep\nextra\n", }, ) ], usage={}, ) return LLMResponse(content="done", tool_calls=[], usage={}) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = Tools() runner = AgentRunner() result = await runner.run(make_run_spec(provider, initial_messages=[{"role": "user", "content": "edit a file"}], tools=tools, model="test-model", max_iterations=2, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, workspace=tmp_path, hook=FileEditActivityHook(on_progress=progress_cb, workspace=tmp_path), )) assert result.final_content == "done" assert any( event["tool"] == "edit_file" and not event["approximate"] and event["phase"] == "end" and event["added"] == 2 and event["deleted"] == 1 and event["diff"]["format"] == "unified" for event in progress_events ) provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_marks_file_edit_activity_failed_when_tool_errors(tmp_path): provider = MagicMock() provider.supports_progress_deltas = True call_count = 0 progress_events: list[dict] = [] async def progress_cb(content, *, file_edit_events=None, **kwargs): if file_edit_events: progress_events.extend(file_edit_events) tool = WriteFileTool(workspace=tmp_path) class Tools: def get_definitions(self): return [{"type": "function", "function": {"name": "write_file"}}] def prepare_call(self, name, params): return tool, params, None async def chat_stream_with_retry(**kwargs): nonlocal call_count call_count += 1 if call_count == 1: return LLMResponse( content=None, tool_calls=[ ToolCallRequest( id="call-write", name="write_file", arguments={"path": "aborted.txt"}, ) ], usage={}, ) return LLMResponse(content="done", tool_calls=[], usage={}) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = Tools() runner = AgentRunner() result = await runner.run(make_run_spec(provider, initial_messages=[{"role": "user", "content": "write a file"}], tools=tools, model="test-model", max_iterations=2, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, workspace=tmp_path, hook=FileEditActivityHook(on_progress=progress_cb, workspace=tmp_path), )) assert result.stop_reason == "completed" assert progress_events[-1]["path"] == "aborted.txt" assert progress_events[-1]["phase"] == "error" assert progress_events[-1]["status"] == "error" provider.chat_with_retry.assert_not_awaited() @pytest.mark.asyncio async def test_runner_marks_file_edit_activity_failed_when_cancelled(tmp_path): provider = MagicMock() provider.supports_progress_deltas = True progress_events: list[dict] = [] executing = asyncio.Event() target = tmp_path / "cancelled.txt" target.write_text("old\n", encoding="utf-8") async def progress_cb(content, *, file_edit_events=None, **kwargs): if file_edit_events: progress_events.extend(file_edit_events) class SlowWriteTool(WriteFileTool): async def execute(self, path=None, content=None, **kwargs): executing.set() await asyncio.sleep(60) return "ok" tool = SlowWriteTool(workspace=tmp_path) class Tools: def get_definitions(self): return [{"type": "function", "function": {"name": "write_file"}}] def prepare_call(self, name, params): return tool, params, None async def chat_stream_with_retry(**kwargs): return LLMResponse( content=None, tool_calls=[ ToolCallRequest( id="call-write", name="write_file", arguments={"path": "cancelled.txt", "content": "new\n"}, ) ], usage={}, ) provider.chat_stream_with_retry = chat_stream_with_retry provider.chat_with_retry = AsyncMock() tools = Tools() runner = AgentRunner() task = asyncio.create_task(runner.run(make_run_spec(provider, initial_messages=[{"role": "user", "content": "write a file"}], tools=tools, model="test-model", max_iterations=2, max_tool_result_chars=_MAX_TOOL_RESULT_CHARS, progress_callback=progress_cb, workspace=tmp_path, hook=FileEditActivityHook(on_progress=progress_cb, workspace=tmp_path), ))) await asyncio.wait_for(executing.wait(), timeout=1) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert [event["phase"] for event in progress_events] == ["start", "error"] assert progress_events[-1]["path"] == "cancelled.txt" assert progress_events[-1]["status"] == "error" assert progress_events[-1]["error"] == "Task interrupted before this tool finished." provider.chat_with_retry.assert_not_awaited()