fix(gateway): make resource teardown cancellation-safe

nanobot/agent/loop.py

@@ -399,6 +399,7 @@ class AgentLoop:
         self._runtime_context_providers: list[RuntimeContextProvider] = []
         self._active_tasks: dict[str, set[asyncio.Task[Any]]] = {}
         self._background_tasks: set[asyncio.Task[Any]] = set()
+        self._close_mcp_lock = asyncio.Lock()
         self._session_locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
             weakref.WeakValueDictionary()
         )
@@ -1338,11 +1339,42 @@ class AgentLoop:
                 await self._publish_next_deferred_automation_turn(session_key)
 
     async def close_mcp(self) -> None:
-        """Drain background work, stop exec sessions, then close MCP connections."""
-        if self._background_tasks:
-            await asyncio.gather(*self._background_tasks, return_exceptions=True)
-            self._background_tasks.clear()
+        """Stop active work, then close exec, subagent, and MCP resources.
+
+        Resource teardown must still run if cancellation interrupts task draining.
+        Gateway shutdown deliberately bounds this coroutine, so keeping the cleanup
+        phase in ``finally`` prevents a timed-out background task from leaving
+        subprocess transports alive after the event loop closes.
+        """
+        # The agent loop closes itself from ``run()`` while gateway shutdown also
+        # performs a guaranteed final close. Serialize those owners so they cannot
+        # tear down the same subprocess transports concurrently.
+        close_lock = getattr(self, "_close_mcp_lock", None)
+        if close_lock is None:
+            close_lock = self._close_mcp_lock = asyncio.Lock()
+        async with close_lock:
+            await self._close_mcp_unlocked()
+
+    async def _close_mcp_unlocked(self) -> None:
         errors: list[BaseException] = []
+        active_task_groups = getattr(self, "_active_tasks", {})
+        active_tasks = tuple({task for tasks in active_task_groups.values() for task in tasks})
+        active_task_groups.clear()
+        current_task = asyncio.current_task()
+        active_tasks = tuple(task for task in active_tasks if task is not current_task)
+        for task in active_tasks:
+            if not task.done():
+                task.cancel()
+        try:
+            if active_tasks:
+                await asyncio.gather(*active_tasks, return_exceptions=True)
+            if self._background_tasks:
+                await asyncio.gather(*self._background_tasks, return_exceptions=True)
+        except BaseException as exc:
+            errors.append(exc)
+        finally:
+            self._background_tasks.clear()
+
         cleanup_steps = (
             self.subagents.close,
             self._exec_session_manager.close_all,

nanobot/cli/gateway_runtime.py

@@ -206,7 +206,6 @@ async def _close_gateway_runtime(
     channels: Any,
     tasks: list[asyncio.Task[Any]],
     runtime_tasks: asyncio.Future[list[Any]] | None,
-    runtime_tasks_drained: bool,
     *,
     task_wait_timeout: float = 15.0,
     close_timeout: float = 15.0,
@@ -230,16 +229,26 @@ async def _close_gateway_runtime(
     for task in tasks:
         if not task.done():
             task.cancel()
+    pending: set[asyncio.Task[Any]] = set()
     if tasks:
         # Bounded: a coroutine that swallows cancellation (e.g. an SDK reconnect
         # loop) must not hold the stop open until systemd's timeout kills the
         # cgroup. Anything still pending is abandoned and closed underneath.
-        await asyncio.wait(tasks, timeout=task_wait_timeout)
+        _done, pending = await asyncio.wait(tasks, timeout=task_wait_timeout)
+        # A task can swallow the first cancellation while unwinding. Re-cancel
+        # timed-out tasks so an agent loop stuck draining background work reaches
+        # its resource-cleanup phase before the explicit final close below.
+        for task in pending:
+            task.cancel()
+    if runtime_tasks is not None and not runtime_tasks.done():
+        runtime_tasks.cancel()
     try:
         await asyncio.wait_for(agent.close_mcp(), timeout=close_timeout)
     except BaseException as exc:  # noqa: BLE001 - shutdown must proceed
         logger.warning("Gateway shutdown: agent resource cleanup incomplete: {}", exc)
-    if runtime_tasks is not None and not runtime_tasks_drained:
+    # Retrieving an already-finished gather prevents noisy unhandled exceptions,
+    # but never wait for it here: its children were bounded individually above.
+    if runtime_tasks is not None and runtime_tasks.done():
         with suppress(asyncio.CancelledError, Exception):
             await runtime_tasks
 
@@ -777,7 +786,6 @@ def _run_gateway(
         tasks: list[asyncio.Task[Any]] = []
         shutdown_task: asyncio.Task[Any] | None = None
         runtime_tasks: asyncio.Future[list[Any]] | None = None
-        runtime_tasks_drained = False
         shutdown_event = asyncio.Event()
         cli_terminal._ensure_interactive_tty_mode()
         restore_shutdown_handlers = _install_gateway_shutdown_handlers(
@@ -829,7 +837,6 @@ def _run_gateway(
                 return_when=asyncio.FIRST_COMPLETED,
             )
             if runtime_tasks in done:
-                runtime_tasks_drained = True
                 await runtime_tasks
             else:
                 runtime_tasks.cancel()
@@ -850,9 +857,7 @@ def _run_gateway(
                 agent.stop()
                 # Cancel runtime tasks first, then deterministically close
                 # exec/MCP resources while the event loop is still alive.
-                await _close_gateway_runtime(
-                    agent, channels, tasks, runtime_tasks, runtime_tasks_drained
-                )
+                await _close_gateway_runtime(agent, channels, tasks, runtime_tasks)
                 # Flush all cached sessions to durable storage before exit.
                 # This prevents data loss on filesystems with write-back
                 # caching (rclone VFS, NFS, FUSE mounts, etc.).