Merge remote-tracking branch 'origin/main' into codex/review-pr-3894

# Conflicts:
#	tests/utils/test_webui_transcript.py

tests/utils/test_artifacts.py

@@ -10,8 +10,6 @@ from nanobot.config.loader import set_config_path
 from nanobot.utils.artifacts import (
     ArtifactError,
     decode_image_data_url,
-    generated_image_paths_from_messages,
-    generated_image_tool_result,
     store_generated_image_artifact,
 )
 
@@ -66,22 +64,3 @@ def test_store_generated_image_artifact_rejects_unsafe_save_dir(tmp_path: Path)
             model="m",
             save_dir="../outside",
         )
-
-
-def test_generated_image_paths_from_tool_results() -> None:
-    result = generated_image_tool_result(
-        [
-            {"id": "img_1", "path": "/tmp/one.png"},
-            {"id": "img_2", "path": "/tmp/two.png"},
-        ]
-    )
-    payload = json.loads(result)
-
-    assert generated_image_paths_from_messages(
-        [
-            {"role": "tool", "name": "generate_image", "content": result},
-            {"role": "tool", "name": "other", "content": result},
-        ]
-    ) == ["/tmp/one.png", "/tmp/two.png"]
-    assert "runtime attaches generated images automatically" in payload["next_step"]
-    assert "Do not call message" in payload["next_step"]

tests/utils/test_file_edit_events.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import asyncio
 from pathlib import Path
+from types import SimpleNamespace
 
 from nanobot.utils.file_edit_events import (
     build_file_edit_end_event,
@@ -8,6 +10,7 @@ from nanobot.utils.file_edit_events import (
     line_diff_stats,
     prepare_file_edit_tracker,
     read_file_snapshot,
+    StreamingFileEditTracker,
 )
 
 
@@ -20,6 +23,10 @@ def test_line_diff_stats_normalizes_crlf() -> None:
     assert line_diff_stats("a\r\nb\r\n", "a\nb\nc\n") == (1, 0)
 
 
+def test_line_diff_stats_counts_new_file_crlf_lines_once() -> None:
+    assert line_diff_stats("", "a\r\nb\r\n") == (2, 0)
+
+
 def test_write_file_start_predicts_and_end_calibrates_exact_diff(tmp_path: Path) -> None:
     target = tmp_path / "notes.txt"
     target.write_text("old\nkeep\n", encoding="utf-8")
@@ -39,6 +46,7 @@ def test_write_file_start_predicts_and_end_calibrates_exact_diff(tmp_path: Path)
         "call_id": "call-write",
         "tool": "write_file",
         "path": "notes.txt",
+        "absolute_path": (tmp_path / "notes.txt").resolve().as_posix(),
         "phase": "start",
         "added": 2,
         "deleted": 1,
@@ -73,6 +81,307 @@ def test_binary_file_is_reported_but_not_counted(tmp_path: Path) -> None:
     assert (event["added"], event["deleted"]) == (0, 0)
 
 
+def test_oversized_write_file_end_uses_known_content_for_exact_count(tmp_path: Path) -> None:
+    target = tmp_path / "large.txt"
+    params = {"path": "large.txt", "content": "x" * (2 * 1024 * 1024 + 1)}
+    tracker = prepare_file_edit_tracker(
+        call_id="call-large",
+        tool_name="write_file",
+        tool=None,
+        workspace=tmp_path,
+        params=params,
+    )
+
+    assert tracker is not None
+    target.write_text(params["content"], encoding="utf-8")
+    event = build_file_edit_end_event(tracker, params)
+    assert event.get("binary") is not True
+    assert event["added"] == 1
+    assert event["deleted"] == 0
+
+
+def test_streaming_write_file_tracker_emits_live_line_counts(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"path":"notes.md","content":"',
+        })
+        await tracker.update({
+            "index": 0,
+            "arguments_delta": "line\\n" * 24,
+        })
+
+    asyncio.run(run())
+
+    assert events[0] == {
+        "version": 1,
+        "call_id": "call-live",
+        "tool": "write_file",
+        "path": "notes.md",
+        "absolute_path": (tmp_path / "notes.md").resolve().as_posix(),
+        "phase": "start",
+        "added": 0,
+        "deleted": 0,
+        "approximate": True,
+        "status": "editing",
+    }
+    assert events[-1]["path"] == "notes.md"
+    assert events[-1]["status"] == "editing"
+    assert events[-1]["approximate"] is True
+    assert events[-1]["added"] == 24
+    assert events[-1]["deleted"] == 0
+
+
+def test_streaming_write_file_tracker_emits_pending_before_path(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"content":"line\\n',
+        })
+        await tracker.update({
+            "index": 0,
+            "arguments_delta": 'more\\n","path":"late.md"',
+        })
+
+    asyncio.run(run())
+
+    assert events[0] == {
+        "version": 1,
+        "call_id": "call-live",
+        "tool": "write_file",
+        "path": "",
+        "phase": "start",
+        "added": 1,
+        "deleted": 0,
+        "approximate": True,
+        "status": "editing",
+        "pending": True,
+    }
+    assert events[-1]["path"] == "late.md"
+    assert events[-1].get("pending") is not True
+    assert events[-1]["added"] == 2
+
+
+def test_streaming_write_file_tracker_flushes_small_pending_count(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"path":"small.md","content":"one\\n',
+        })
+        await tracker.flush()
+
+    asyncio.run(run())
+    assert events
+    assert events[-1]["path"] == "small.md"
+    assert events[-1]["added"] == 1
+
+
+def test_streaming_write_file_tracker_normalizes_crlf_line_counts(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"path":"windows.txt","content":"one\\r\\ntwo\\r\\n',
+        })
+        await tracker.flush()
+
+    asyncio.run(run())
+    assert events[-1]["path"] == "windows.txt"
+    assert events[-1]["added"] == 2
+
+
+def test_streaming_write_file_tracker_counts_unicode_escaped_newlines(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"path":"unicode.txt","content":"one\\u000atwo',
+        })
+        await tracker.flush()
+
+    asyncio.run(run())
+    assert events[-1]["path"] == "unicode.txt"
+    assert events[-1]["added"] == 2
+
+
+def test_streaming_edit_file_tracker_emits_live_line_counts(tmp_path: Path) -> None:
+    target = tmp_path / "notes.md"
+    target.write_text("old\nkeep\n", encoding="utf-8")
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-edit",
+            "name": "edit_file",
+            "arguments_delta": '{"path":"notes.md","old_text":"old\\nkeep","new_text":"',
+        })
+        await tracker.update({
+            "index": 0,
+            "arguments_delta": "new\\nkeep\\nextra\\n" * 8,
+        })
+
+    asyncio.run(run())
+
+    assert events[0] == {
+        "version": 1,
+        "call_id": "call-edit",
+        "tool": "edit_file",
+        "path": "notes.md",
+        "absolute_path": (tmp_path / "notes.md").resolve().as_posix(),
+        "phase": "start",
+        "added": 0,
+        "deleted": 2,
+        "approximate": True,
+        "status": "editing",
+    }
+    assert events[-1]["path"] == "notes.md"
+    assert events[-1]["status"] == "editing"
+    assert events[-1]["approximate"] is True
+    assert events[-1]["added"] == 24
+    assert events[-1]["deleted"] == 2
+
+
+def test_streaming_tracker_applies_canonical_call_id_to_final_tool(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "name": "write_file",
+            "arguments_delta": '{"path":"matched.md","content":"one\\n',
+        })
+        final = SimpleNamespace(
+            id="provider-final-id",
+            name="write_file",
+            arguments={"path": "matched.md", "content": "one\n"},
+        )
+        tracker.apply_final_call_ids([final])
+        assert final.id == "idx:0"
+
+    asyncio.run(run())
+
+
+def test_streaming_edit_file_tracker_flushes_small_pending_count(tmp_path: Path) -> None:
+    target = tmp_path / "small.py"
+    target.write_text("old\n", encoding="utf-8")
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-edit",
+            "name": "edit_file",
+            "arguments_delta": '{"path":"small.py","old_text":"old\\n","new_text":"new\\nextra',
+        })
+        await tracker.flush()
+
+    asyncio.run(run())
+    assert events
+    assert events[-1]["path"] == "small.py"
+    assert events[-1]["added"] == 2
+    assert events[-1]["deleted"] == 1
+
+
+def test_streaming_write_file_tracker_errors_unmatched_live_edits(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "call-live",
+            "name": "write_file",
+            "arguments_delta": '{"path":"aborted.md","content":"one\\n',
+        })
+        await tracker.error_unmatched([], "Tool call did not complete.")
+
+    asyncio.run(run())
+    assert events[-1]["path"] == "aborted.md"
+    assert events[-1]["phase"] == "error"
+    assert events[-1]["status"] == "error"
+
+
+def test_streaming_write_file_tracker_keeps_matched_final_tool_call(tmp_path: Path) -> None:
+    events: list[dict] = []
+
+    async def emit(batch: list[dict]) -> None:
+        events.extend(batch)
+
+    async def run() -> None:
+        tracker = StreamingFileEditTracker(workspace=tmp_path, tools={}, emit=emit)
+        await tracker.update({
+            "index": 0,
+            "call_id": "idx-only",
+            "name": "write_file",
+            "arguments_delta": '{"path":"matched.md","content":"one\\n',
+        })
+        await tracker.error_unmatched([
+            SimpleNamespace(
+                id="final-call",
+                name="write_file",
+                arguments={"path": "matched.md", "content": "one\n"},
+            )
+        ], "Tool call did not complete.")
+
+    asyncio.run(run())
+    assert events
+    assert all(event["status"] == "editing" for event in events)
+
+
 def test_untracked_tools_do_not_prepare_file_edit_tracker(tmp_path: Path) -> None:
     assert prepare_file_edit_tracker(
         call_id="call-exec",

tests/utils/test_webui_compat_imports.py

@@ -0,0 +1,14 @@
+import importlib
+
+from nanobot.session import webui_turns
+from nanobot.webui import thread_disk, transcript
+
+
+def test_legacy_webui_utils_imports_resolve_to_new_modules() -> None:
+    legacy_thread_disk = importlib.import_module("nanobot.utils.webui_thread_disk")
+    legacy_transcript = importlib.import_module("nanobot.utils.webui_transcript")
+    legacy_turn_helpers = importlib.import_module("nanobot.utils.webui_turn_helpers")
+
+    assert legacy_thread_disk.delete_webui_thread is thread_disk.delete_webui_thread
+    assert legacy_transcript.append_transcript_object is transcript.append_transcript_object
+    assert legacy_turn_helpers.mark_webui_session is webui_turns.mark_webui_session

tests/utils/test_webui_sidebar_state.py

@@ -0,0 +1,73 @@
+import json
+
+from nanobot.webui.sidebar_state import (
+    default_webui_sidebar_state,
+    read_webui_sidebar_state,
+    webui_sidebar_state_path,
+    write_webui_sidebar_state,
+)
+
+
+def test_sidebar_state_defaults_when_file_missing(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+
+    state = read_webui_sidebar_state()
+
+    assert state == default_webui_sidebar_state()
+    assert webui_sidebar_state_path() == tmp_path / "webui" / "sidebar-state.json"
+
+
+def test_sidebar_state_normalizes_old_or_partial_payload(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    path = webui_sidebar_state_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(
+            {
+                "pinned_keys": ["websocket:a", "websocket:a", "", 123],
+                "archived_keys": ["websocket:b"],
+                "title_overrides": {"websocket:a": "  Release notes  ", "bad": ""},
+                "tags_by_key": {"websocket:a": ["work", "work", ""]},
+                "collapsed_groups": {"Earlier": 1},
+                "view": {"density": "tiny", "show_archived": True, "sort": "nope"},
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    state = read_webui_sidebar_state()
+
+    assert state["schema_version"] == 1
+    assert state["pinned_keys"] == ["websocket:a"]
+    assert state["archived_keys"] == ["websocket:b"]
+    assert state["title_overrides"] == {"websocket:a": "Release notes"}
+    assert state["tags_by_key"] == {"websocket:a": ["work"]}
+    assert state["collapsed_groups"] == {"Earlier": True}
+    assert state["view"] == {
+        "density": "comfortable",
+        "show_previews": False,
+        "show_timestamps": False,
+        "show_archived": True,
+        "sort": "updated_desc",
+    }
+
+
+def test_sidebar_state_write_is_scoped_to_config_data_dir(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+
+    state = write_webui_sidebar_state(
+        {
+            "pinned_keys": ["websocket:a"],
+            "archived_keys": ["websocket:b"],
+            "title_overrides": {"websocket:a": "Release"},
+            "view": {"density": "compact", "show_previews": True},
+        }
+    )
+
+    assert state["pinned_keys"] == ["websocket:a"]
+    assert state["archived_keys"] == ["websocket:b"]
+    assert state["title_overrides"] == {"websocket:a": "Release"}
+    assert state["view"]["density"] == "compact"
+    assert state["view"]["show_previews"] is True
+    assert webui_sidebar_state_path().is_file()
+    assert read_webui_sidebar_state()["pinned_keys"] == ["websocket:a"]

tests/utils/test_webui_thread_disk.py

@@ -2,8 +2,8 @@
 
 from __future__ import annotations
 
-from nanobot.utils.webui_thread_disk import delete_webui_thread, webui_thread_file_path
-from nanobot.utils.webui_transcript import append_transcript_object, webui_transcript_path
+from nanobot.webui.thread_disk import delete_webui_thread, webui_thread_file_path
+from nanobot.webui.transcript import append_transcript_object, webui_transcript_path
 
 
 def test_delete_webui_thread_removes_legacy_json_and_transcript(tmp_path, monkeypatch) -> None:

tests/utils/test_webui_transcript.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from nanobot.utils.webui_transcript import (
+from nanobot.webui.transcript import (
     WEBUI_TRANSCRIPT_SCHEMA_VERSION,
     append_transcript_object,
     read_transcript_lines,
@@ -145,8 +145,203 @@ def test_replay_tool_events_dedupes_finish_after_start() -> None:
     ]
 
 
+def test_replay_file_edit_progress_merges_after_interleaved_activity(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:t-file-progress"
+    for ev in (
+        {"event": "user", "chat_id": "t-file-progress", "text": "edit"},
+        {
+            "event": "message",
+            "chat_id": "t-file-progress",
+            "text": 'write_file({"path":"foo.txt"})',
+            "kind": "tool_hint",
+        },
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-progress",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-write",
+                    "tool": "write_file",
+                    "path": "foo.txt",
+                    "phase": "start",
+                    "added": 12,
+                    "deleted": 0,
+                    "approximate": True,
+                    "status": "editing",
+                },
+            ],
+        },
+        {
+            "event": "message",
+            "chat_id": "t-file-progress",
+            "text": "still working",
+            "kind": "progress",
+        },
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-progress",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-write",
+                    "tool": "write_file",
+                    "path": "foo.txt",
+                    "phase": "end",
+                    "added": 30,
+                    "deleted": 0,
+                    "approximate": False,
+                    "status": "done",
+                },
+            ],
+        },
+    ):
+        append_transcript_object(key, ev)
+
+    msgs = replay_transcript_to_ui_messages(read_transcript_lines(key))
+    file_edit_messages = [msg for msg in msgs if msg.get("fileEdits")]
+
+    assert len(file_edit_messages) == 1
+    assert file_edit_messages[0]["fileEdits"] == [
+        {
+            "version": 1,
+            "call_id": "call-write",
+            "tool": "write_file",
+            "path": "foo.txt",
+            "phase": "end",
+            "added": 30,
+            "deleted": 0,
+            "approximate": False,
+            "status": "done",
+        },
+    ]
+
+
+def test_replay_file_edit_pending_placeholder_upgrades_to_path(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:t-file-pending"
+    for ev in (
+        {"event": "user", "chat_id": "t-file-pending", "text": "write"},
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-pending",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-write",
+                    "tool": "write_file",
+                    "path": "",
+                    "phase": "start",
+                    "added": 1,
+                    "deleted": 0,
+                    "approximate": True,
+                    "status": "editing",
+                    "pending": True,
+                },
+            ],
+        },
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-pending",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-write",
+                    "tool": "write_file",
+                    "path": "foo.txt",
+                    "phase": "start",
+                    "added": 12,
+                    "deleted": 0,
+                    "approximate": True,
+                    "status": "editing",
+                },
+            ],
+        },
+    ):
+        append_transcript_object(key, ev)
+
+    msgs = replay_transcript_to_ui_messages(read_transcript_lines(key))
+    file_edit_messages = [msg for msg in msgs if msg.get("fileEdits")]
+
+    assert len(file_edit_messages) == 1
+    assert file_edit_messages[0]["fileEdits"] == [
+        {
+            "version": 1,
+            "call_id": "call-write",
+            "tool": "write_file",
+            "path": "foo.txt",
+            "phase": "start",
+            "added": 12,
+            "deleted": 0,
+            "approximate": True,
+            "status": "editing",
+        },
+    ]
+
+
+def test_replay_keeps_new_file_edit_after_reasoning_in_order(tmp_path, monkeypatch) -> None:
+    monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
+    key = "websocket:t-file-order"
+    for ev in (
+        {"event": "user", "chat_id": "t-file-order", "text": "edit"},
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-order",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-one",
+                    "tool": "write_file",
+                    "path": "one.txt",
+                    "phase": "start",
+                    "added": 10,
+                    "deleted": 0,
+                    "approximate": True,
+                    "status": "editing",
+                },
+            ],
+        },
+        {"event": "reasoning_delta", "chat_id": "t-file-order", "text": "Check next."},
+        {"event": "reasoning_end", "chat_id": "t-file-order"},
+        {
+            "event": "file_edit",
+            "chat_id": "t-file-order",
+            "edits": [
+                {
+                    "version": 1,
+                    "call_id": "call-two",
+                    "tool": "write_file",
+                    "path": "two.txt",
+                    "phase": "start",
+                    "added": 20,
+                    "deleted": 0,
+                    "approximate": True,
+                    "status": "editing",
+                },
+            ],
+        },
+    ):
+        append_transcript_object(key, ev)
+
+    msgs = replay_transcript_to_ui_messages(read_transcript_lines(key))
+
+    assert [msg.get("fileEdits", [{}])[0].get("path") if msg.get("fileEdits") else msg.get("reasoning") for msg in msgs[1:]] == [
+        "one.txt",
+        "Check next.",
+        "two.txt",
+    ]
+    file_edit_segments = [
+        msg.get("activitySegmentId")
+        for msg in msgs
+        if msg.get("fileEdits")
+    ]
+    assert len(file_edit_segments) == 2
+    assert file_edit_segments[0] != file_edit_segments[1]
+
+
 def test_build_response_schema(monkeypatch, tmp_path) -> None:
-    from nanobot.utils.webui_transcript import build_webui_thread_response
+    from nanobot.webui.transcript import build_webui_thread_response
 
     monkeypatch.setattr("nanobot.config.paths.get_data_dir", lambda: tmp_path)
     key = "websocket:t3"

tests/utils/test_webui_turn_helpers.py

@@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock
 import pytest
 
 from nanobot.bus.events import InboundMessage
-from nanobot.utils import webui_turn_helpers as wth
+from nanobot.session import webui_turns as wth
 
 
 @pytest.fixture(autouse=True)