feat(runtime): add user-controlled turn recovery

This commit is contained in:
Xubin Ren
2026-08-24 00:58:04 +08:00
parent ffa58aa5ef
commit 12029f8812
60 changed files with 4027 additions and 169 deletions
+11
View File
@@ -104,6 +104,9 @@ class ChannelManager:
webui_mcp_runtime_status: Callable[[], Mapping[str, str]] | None = None,
webui_mcp_reload: Callable[[], Awaitable[dict[str, Any]]] | None = None,
webui_skill_state_action: Callable[[set[str]], None] | None = None,
webui_recovery_action: (
Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]] | None
) = None,
config_path: Path | None = None,
):
if config_path is None:
@@ -126,6 +129,7 @@ class ChannelManager:
self._webui_mcp_runtime_status = webui_mcp_runtime_status
self._webui_mcp_reload = webui_mcp_reload
self._webui_skill_state_action = webui_skill_state_action
self._webui_recovery_action = webui_recovery_action
self.channels: dict[str, BaseChannel] = {}
self._channel_owners: dict[str, str] = {}
self._channel_runtime_specs: dict[str, tuple[str, str]] = {}
@@ -197,6 +201,7 @@ class ChannelManager:
mcp_runtime_status=self._webui_mcp_runtime_status,
mcp_reload=self._webui_mcp_reload,
skill_state_action=self._webui_skill_state_action,
recovery_action=self._webui_recovery_action,
logger=logger,
)
kwargs["gateway"] = gateway
@@ -615,6 +620,12 @@ class ChannelManager:
if target is None:
logger.warning("Restart notice target channel is not enabled: {}", notice.channel)
return
if notice.channel == "websocket":
# Reconnect and recovery are already represented by WebSocket
# protocol state. A generic restart-complete notice must not
# masquerade as a recovery transition and overwrite a real
# awaiting-user checkpoint in connected clients.
return
while not target.is_running:
remaining = deadline - loop.time()
+30
View File
@@ -32,6 +32,7 @@ from nanobot.bus.outbound_events import (
GoalStateSyncEvent,
GoalStatusEvent,
ProgressEvent,
RecoveryStateEvent,
RuntimeModelUpdatedEvent,
SessionUpdatedEvent,
TurnEndEvent,
@@ -55,6 +56,7 @@ from nanobot.security.workspace_access import (
)
from nanobot.session.goal_state import goal_state_ws_blob
from nanobot.session.model_selection import model_preset_from_metadata
from nanobot.session.recovery import recovery_state_from_metadata
from nanobot.session.webui_turns import (
clear_websocket_turn_if_current,
clear_websocket_turns,
@@ -453,6 +455,9 @@ class WebSocketChannel(BaseChannel):
self.logger.warning("ignoring invalid model preset metadata for chat_id={}", chat_id)
fields["model_preset"] = None
if isinstance(metadata, dict):
recovery_state = recovery_state_from_metadata(metadata)
if recovery_state is not None:
fields["recovery_state"] = recovery_state
usage = metadata.get("_last_usage")
if isinstance(usage, dict):
sanitized_usage: dict[str, int | float] = {}
@@ -1740,6 +1745,10 @@ class WebSocketChannel(BaseChannel):
provenance=event.provenance,
)
return
if isinstance(event, RecoveryStateEvent):
if conns:
await self.send_recovery_state(msg.chat_id, event)
return
if isinstance(event, GoalStateSyncEvent):
if conns:
await self.send_goal_state(msg.chat_id, event.goal_state or {"active": False})
@@ -2057,6 +2066,27 @@ class WebSocketChannel(BaseChannel):
for connection in conns:
await self._safe_send_to(connection, raw, label=" turn_end ")
async def send_recovery_state(
self,
chat_id: str,
event: RecoveryStateEvent,
) -> None:
"""Publish one structured recovery transition without chat pollution."""
body: dict[str, Any] = {
"event": "recovery_state",
"chat_id": chat_id,
"status": event.status,
"recovery_id": event.recovery_id,
"attempts": event.attempts,
}
if event.reason:
body["reason"] = event.reason
if event.can_continue is not None:
body["can_continue"] = event.can_continue
raw = json.dumps(body, ensure_ascii=False)
for connection in list(self._subs.get(chat_id, ())):
await self._safe_send_to(connection, raw, label=" recovery_state ")
async def send_goal_state(self, chat_id: str, blob: dict[str, Any]) -> None:
"""Push persisted goal-state snapshot for *chat_id* (multi-chat isolation)."""
conns = list(self._subs.get(chat_id, ()))
@@ -27,6 +27,7 @@ from nanobot.bus.outbound_events import (
GoalStateSyncEvent,
GoalStatusEvent,
ProgressEvent,
RecoveryStateEvent,
RuntimeModelUpdatedEvent,
SessionUpdatedEvent,
TurnEndEvent,
@@ -2720,6 +2721,39 @@ async def test_send_turn_end_emits_turn_end_event() -> None:
]
@pytest.mark.asyncio
async def test_recovery_state_is_a_structured_event_not_assistant_text() -> None:
bus = MagicMock()
channel = WebSocketChannel(
{"enabled": True, "allowFrom": ["*"]},
bus,
gateway=_basic_handler(bus),
)
mock_ws = AsyncMock()
channel._attach(mock_ws, "chat-1")
await channel.send(OutboundMessage(
channel="websocket",
chat_id="chat-1",
content="",
event=RecoveryStateEvent(
status="awaiting_user",
recovery_id="recovery-1",
reason="tool_state_unknown",
attempts=1,
),
))
assert _sent_ws_payloads(mock_ws) == [{
"event": "recovery_state",
"chat_id": "chat-1",
"status": "awaiting_user",
"recovery_id": "recovery-1",
"reason": "tool_state_unknown",
"attempts": 1,
}]
@pytest.mark.asyncio
async def test_system_command_turn_end_only_refreshes_session_metadata() -> None:
bus = MagicMock()
@@ -83,6 +83,7 @@ def _make_handler(
channel_feature_action: Any | None = None,
channel_runtime_status: Any | None = None,
mcp_reload: Any | None = None,
recovery_action: Any | None = None,
) -> GatewayServices:
config = WebSocketConfig.model_validate(cfg) if isinstance(cfg, dict) else cfg
workspace = workspace_path or Path.cwd()
@@ -103,6 +104,7 @@ def _make_handler(
channel_feature_action=channel_feature_action,
channel_runtime_status=channel_runtime_status,
mcp_reload=mcp_reload,
recovery_action=recovery_action,
)
@@ -121,6 +123,7 @@ def _ch(
channel_feature_action: Any | None = None,
channel_runtime_status: Any | None = None,
mcp_reload: Any | None = None,
recovery_action: Any | None = None,
**extra: Any,
) -> WebSocketChannel:
cfg: dict[str, Any] = {
@@ -145,6 +148,7 @@ def _ch(
channel_feature_action=channel_feature_action,
channel_runtime_status=channel_runtime_status,
mcp_reload=mcp_reload,
recovery_action=recovery_action,
)
return InProcessHttpChannel(cfg, bus, gateway=gateway)
@@ -3242,6 +3246,28 @@ async def _webui_mutate(
)
@pytest.mark.asyncio
async def test_recovery_mutation_uses_authenticated_websocket_action(bus: MagicMock) -> None:
recovery_action = AsyncMock(return_value={
"status": "resuming",
"recovery_id": "recovery-1",
})
channel = _ch(bus, recovery_action=recovery_action)
response = await _webui_mutate(
channel,
"recovery.continue",
{"chat_id": "chat-1", "recovery_id": "recovery-1"},
)
assert response.status_code == 200
assert response.json()["status"] == "resuming"
recovery_action.assert_awaited_once_with(
"continue",
{"chat_id": "chat-1", "recovery_id": "recovery-1"},
)
@pytest.mark.asyncio
async def test_workspace_folder_picker_is_local_authenticated_mutation(
bus: MagicMock,