Grayscale/nanobot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gateway): harden shared client lifecycle

Browse Source
This commit is contained in:
Xubin Ren
2026-08-17 20:56:10 +08:00
parent 19be5be1c0
commit cd6a11b3c5
11 changed files with 468 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanobot/cli/gateway.py
+46 -2
View File
@@ -128,6 +128,10 @@ def create_gateway_app(
console.print(f"Port: {status.port}")
if status.started_at is not None:
console.print(f"Started At: {status.started_at}")
if status.running:
console.print(f"Launch Mode: {status.launch_mode}")
console.print(f"Lifetime: {status.lifetime}")
console.print(f"Clients: {status.clients}")
console.print(f"State: {status.state_path}")
console.print(f"Logs: {status.log_path}")
@@ -176,6 +180,35 @@ def create_gateway_app(
loaded_config=cfg,
)
)
if (
result.message == "gateway_already_running"
and result.status.launch_mode == "foreground"
):
console.print(
"[yellow]Gateway is already running in the foreground; "
"an attached process cannot be detached in place.[/yellow]"
)
console.print(
"[dim]Stop it in its current terminal, then run "
"`nanobot gateway --background`.[/dim]"
)
print_status(result.status)
raise typer.Exit(1)
if (
result.message == "gateway_already_running"
and result.status.launch_mode == "unknown"
and result.status.lifetime == "explicit"
):
console.print(
"[yellow]Gateway is already running, but this older process did "
"not record whether it is attached or detached.[/yellow]"
)
console.print(
"[dim]Stop it first, then rerun `nanobot gateway --background` "
"to establish an unambiguous lifecycle.[/dim]"
)
print_status(result.status)
raise typer.Exit(1)
promoted = False
if result.ok or result.message == "gateway_already_running":
promoted = GatewayClientLease(
@@ -184,7 +217,7 @@ def create_gateway_app(
).mark_persistent()
if result.ok:
console.print("[green]Gateway started in the background.[/green]")
print_status(result.status)
print_status(runtime.status())
return
if result.message == "gateway_already_running":
if promoted:
@@ -201,7 +234,7 @@ def create_gateway_app(
"[yellow]Gateway is already running in persistent "
"background mode.[/yellow]"
)
print_status(result.status)
print_status(runtime.status())
return
console.print(f"[yellow]Gateway was not started: {result.message}[/yellow]")
print_status(result.status)
@@ -303,6 +336,17 @@ def create_gateway_app(
)
print_status(result.status)
raise typer.Exit(1)
if result.message == "gateway_foreground_restart_required":
console.print(
"[yellow]Gateway is attached to a foreground terminal and cannot "
"be restarted as a background process.[/yellow]"
)
console.print(
"[dim]Restart it in that terminal, or stop it and run "
"`nanobot gateway --background`.[/dim]"
)
print_status(result.status)
raise typer.Exit(1)
console.print(f"[red]Gateway restart failed: {result.message}[/red]")
print_status(result.status)
raise typer.Exit(1)
nanobot/cli/gateway_runtime.py
+19 -1
View File
@@ -389,7 +389,13 @@ def _run_gateway(
# Use the same runtime identity for foreground and managed gateway processes.
from nanobot.config.loader import get_config_path
from nanobot.gateway.runtime import GatewayRuntime, GatewayRuntimePaths, GatewayStartOptions
from nanobot.gateway.runtime import (
GatewayClientLease,
GatewayRuntime,
GatewayRuntimePaths,
GatewayStartOptions,
monitor_gateway_clients,
)
config_path = str(get_config_path().resolve(strict=False))
gateway_workspace = (
@@ -872,6 +878,14 @@ def _run_gateway(
finally:
await mcp_provider.aclose()
async def _monitor_local_clients() -> None:
orphaned = await monitor_gateway_clients(
GatewayClientLease(gateway_runtime, kind="gateway-monitor"),
shutdown_event,
)
if orphaned:
logger.info("Last local client disappeared; stopping on-demand gateway")
tasks = [
asyncio.create_task(
watch_config_file(
@@ -890,6 +904,10 @@ def _run_gateway(
),
name="nanobot-local-triggers",
),
asyncio.create_task(
_monitor_local_clients(),
name="nanobot-gateway-client-monitor",
),
]
if health_server_enabled:
tasks.append(asyncio.create_task(
nanobot/cli/tui_launcher.py
+1 -6
View File
@@ -279,16 +279,13 @@ def _ensure_gateway(
"stop that instance or use `nanobot agent --classic`"
)
result = runtime.start_background(
result = lease.ensure_on_demand_gateway(
GatewayStartOptions(
port=config.gateway.port,
workspace=workspace_override_path,
config_path=str(config_path),
)
)
started_here = result.ok
if started_here:
lease.mark_ephemeral()
if not result.ok and result.message != "gateway_already_running":
raise TuiUnavailableError(
f"could not start the local gateway ({result.message}); "
@@ -309,8 +306,6 @@ def _ensure_gateway(
break
time.sleep(0.1)
if started_here:
runtime.stop(timeout_s=5)
raise TuiUnavailableError(
f"local gateway did not become ready; logs: {result.status.log_path}"
)
nanobot/cli/webui.py
+2 -5
View File
@@ -234,16 +234,13 @@ def webui(
config_path=config_arg,
)
def ensure_shared_gateway(*, client_lease: GatewayClientLease | None = None) -> None:
def ensure_shared_gateway(*, client_lease: GatewayClientLease) -> None:
"""Start or refresh the one managed gateway shared by local clients."""
_prepare_webui_bundle_for_gateway(
runtime_config,
mode="skip" if dev else webui_bundle_mode,
)
result = runtime.start_background(start_options)
started_fresh = result.ok
if started_fresh and client_lease is not None:
client_lease.mark_ephemeral()
result = client_lease.ensure_on_demand_gateway(start_options)
restarted = False
restart_attempted = False
if not result.ok and result.message == "gateway_already_running" and changed_webui:
nanobot/gateway/runtime.py
+169 -16
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import os
@@ -14,7 +15,7 @@ from contextlib import contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Generator, cast
from typing import Any, Generator, Literal, cast
from filelock import FileLock
@@ -28,8 +29,33 @@ from nanobot.process_runtime import (
)
GatewayStartOptions = ProcessStartOptions
GatewayStatus = ProcessStatus
RuntimeResult = ProcessResult
GatewayLaunchMode = Literal["foreground", "background", "unknown"]
GatewayLifetime = Literal["explicit", "on_demand"]
@dataclass(frozen=True)
class GatewayStatus(ProcessStatus):
"""Observable lifecycle state for one shared local gateway."""
launch_mode: GatewayLaunchMode = "unknown"
lifetime: GatewayLifetime = "explicit"
clients: int = 0
@dataclass(frozen=True)
class GatewayLeaseSnapshot:
"""Live local clients and the gateway lifetime they imply."""
auto_stop: bool
clients: int
@dataclass(frozen=True)
class RuntimeResult(ProcessResult):
"""Result of a gateway lifecycle operation."""
status: GatewayStatus
def build_gateway_command(python_executable: str, options: GatewayStartOptions) -> list[str]:
@@ -104,19 +130,78 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]):
def _build_child_command(self, options: ProcessStartOptions) -> list[str]:
return build_gateway_command(self.python_executable, options)
def start_background(self, options: ProcessStartOptions) -> RuntimeResult:
"""Start the gateway detached from the current terminal."""
with self._lifecycle_lock():
return self._start_background(options)
def start_on_demand(self, options: ProcessStartOptions) -> RuntimeResult:
"""Atomically reuse a gateway or start one owned by local client leases."""
with self._lifecycle_lock():
status = self.status()
if status.running:
return RuntimeResult(False, "gateway_already_running", status)
GatewayClientLease(self, kind="gateway-start").mark_ephemeral()
return self._start_background(options)
def _start_background(self, options: ProcessStartOptions) -> RuntimeResult:
result = super()._start_background(options)
if not result.ok:
return self._result(result)
state = self._read_state()
if state and result.status.pid == state.get("pid"):
state["launch_mode"] = "background"
self._write_state(state)
return RuntimeResult(True, result.message, self.status())
def stop(self, *, timeout_s: int = 20) -> RuntimeResult:
"""Stop the gateway recorded by this runtime."""
with self._lifecycle_lock():
return self._result(self._stop(timeout_s=timeout_s))
def status(self, *, reason: str | None = None) -> GatewayStatus:
"""Return process, launch, and client lifetime state in one snapshot."""
process = super().status(reason=reason)
state = self._read_state() if process.running else None
raw_mode = state.get("launch_mode") if state else None
launch_mode: GatewayLaunchMode = (
raw_mode if raw_mode in {"foreground", "background"} else "unknown"
)
lease = GatewayClientLease(self, kind="gateway-status").snapshot()
return GatewayStatus(
running=process.running,
pid=process.pid,
state_path=process.state_path,
log_path=process.log_path,
started_at=process.started_at,
port=process.port,
command=process.command,
reason=process.reason,
launch_mode=launch_mode,
lifetime="on_demand" if lease.auto_stop else "explicit",
clients=lease.clients,
)
@contextmanager
def foreground_instance(self, options: ProcessStartOptions) -> Generator[None]:
"""Publish this foreground gateway while it is available to local clients."""
self._claim_current_process(options)
launch_mode = self._claim_current_process(options)
if launch_mode == "foreground":
GatewayClientLease(self, kind="gateway-foreground").mark_persistent()
try:
yield
finally:
self._release_current_process()
def _claim_current_process(self, options: ProcessStartOptions) -> None:
def _claim_current_process(self, options: ProcessStartOptions) -> GatewayLaunchMode:
with self._lifecycle_lock():
state = self._read_state() or {}
pid = os.getpid()
launch_mode = (
"background"
if state.get("pid") == pid and state.get("launch_mode") == "background"
else "foreground"
)
state.update(
{
"pid": pid,
@@ -128,9 +213,11 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]):
"config_path": options.config_path,
"command": self._build_child_command(options),
"log_path": str(self.paths.log_path),
"launch_mode": launch_mode,
}
)
self._write_state(state)
return launch_mode
def _release_current_process(self) -> None:
with self._lifecycle_lock():
@@ -138,17 +225,24 @@ class GatewayRuntime(ManagedProcessRuntime[ProcessStartOptions]):
if state and self._record_matches_process(state, os.getpid()):
self._clear_state()
def restart(self, options: ProcessStartOptions, *, timeout_s: int = 20) -> ProcessResult:
def restart(self, options: ProcessStartOptions, *, timeout_s: int = 20) -> RuntimeResult:
"""Restart an existing gateway without creating a new persistent instance."""
with self._lifecycle_lock():
status = self.status()
if not status.running:
return ProcessResult(False, "gateway_not_running", status)
return RuntimeResult(False, "gateway_not_running", status)
if status.launch_mode == "foreground":
return RuntimeResult(False, "gateway_foreground_restart_required", status)
stop_result = self._stop(timeout_s=timeout_s)
if not stop_result.ok:
return stop_result
return self._result(stop_result)
return self._start_background(options)
def _result(self, result: ProcessResult) -> RuntimeResult:
status = result.status
gateway_status = status if isinstance(status, GatewayStatus) else self.status()
return RuntimeResult(result.ok, result.message, gateway_status)
class GatewayClientLease:
"""Reference-count an on-demand gateway shared by local interactive clients."""
@@ -180,10 +274,17 @@ class GatewayClientLease:
clients[self.token] = {
"pid": self.pid,
"kind": self.kind,
"identity": self._process_identity(self.pid),
}
self._write_state(state)
self._acquired = True
def ensure_on_demand_gateway(self, options: GatewayStartOptions) -> RuntimeResult:
"""Atomically reuse a gateway or start one owned by local client leases."""
if not self._acquired:
raise RuntimeError("gateway client lease must be acquired before startup")
return self.runtime.start_on_demand(options)
def mark_ephemeral(self) -> None:
"""Mark a gateway started by a client for last-client shutdown."""
with self.lock:
@@ -205,23 +306,43 @@ class GatewayClientLease:
with self.lock:
self.state_path.unlink(missing_ok=True)
def snapshot(self) -> GatewayLeaseSnapshot:
"""Prune dead clients and return current lifetime state."""
with self.lock:
state = self._live_state()
self._write_or_clear(state)
return GatewayLeaseSnapshot(
auto_stop=bool(state.get("auto_stop")),
clients=len(self._clients(state)),
)
def orphaned_on_demand(self) -> bool:
"""Return whether an on-demand gateway has lost every live client."""
snapshot = self.snapshot()
return snapshot.auto_stop and snapshot.clients == 0
def release(self, *, timeout_s: int = 20) -> bool:
"""Release this client and stop an ephemeral gateway when it was the last."""
if not self._acquired:
return False
should_stop = False
with self.lock:
state = self._live_state()
clients = self._clients(state)
clients.pop(self.token, None)
self._acquired = False
if clients or not bool(state.get("auto_stop")):
self._write_or_clear(state)
return False
result = self.runtime.stop(timeout_s=timeout_s)
should_stop = not clients and bool(state.get("auto_stop"))
self._write_or_clear(state)
if not should_stop:
return False
result = self.runtime.stop(timeout_s=timeout_s)
with self.lock:
stopped = result.ok or result.message == "gateway_not_running"
if stopped:
self.state_path.unlink(missing_ok=True)
else:
state = self._live_state()
state["auto_stop"] = True
self._write_state(state)
return stopped
@@ -234,12 +355,27 @@ class GatewayClientLease:
stale.append(token)
continue
record = cast(dict[str, object], value)
if not _pid_is_running(record.get("pid")):
pid = record.get("pid")
identity = record.get("identity")
if (
not isinstance(pid, int)
or not self._process_is_running(pid)
or (identity is not None and identity != self._process_identity(pid))
):
stale.append(token)
for token in stale:
clients.pop(token, None)
return state
def _process_identity(self, pid: int) -> str | int | None:
resolver = getattr(self.runtime, "process_identity", None)
value = resolver(pid) if callable(resolver) else None
return value if isinstance(value, (str, int)) else None
def _process_is_running(self, pid: int) -> bool:
checker = getattr(self.runtime, "process_is_running", None)
return bool(checker(pid)) if callable(checker) else _pid_is_running(pid)
@staticmethod
def _clients(state: dict[str, object]) -> dict[str, object]:
value = state.get("clients")
@@ -284,6 +420,23 @@ class GatewayClientLease:
temporary.unlink(missing_ok=True)
async def monitor_gateway_clients(
lease: GatewayClientLease,
shutdown_event: asyncio.Event,
*,
poll_interval_s: float = 1.0,
) -> bool:
"""Stop waiting when an on-demand gateway loses every live client."""
while not shutdown_event.is_set():
try:
await asyncio.wait_for(shutdown_event.wait(), timeout=poll_interval_s)
except TimeoutError:
if lease.orphaned_on_demand():
shutdown_event.set()
return True
return False
def _instance_suffix(*, workspace: str | None, config_path: str | None) -> str | None:
raw = "|".join(value for value in (workspace, config_path) if value)
if not raw:
@@ -291,11 +444,11 @@ def _instance_suffix(*, workspace: str | None, config_path: str | None) -> str |
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16]
def _pid_is_running(value: object) -> bool:
if not isinstance(value, int) or value <= 0:
def _pid_is_running(pid: int) -> bool:
if pid <= 0:
return False
try:
os.kill(value, 0)
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
nanobot/process_runtime.py
+43 -2
View File
@@ -255,6 +255,14 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
except KeyboardInterrupt:
return 130
def process_identity(self, pid: int) -> str | int | None:
"""Return an identity that changes when an operating-system PID is reused."""
return self._process_identity(pid)
def process_is_running(self, pid: int) -> bool:
"""Return whether the recorded operating-system process is still live."""
return self._is_pid_running(pid)
def _message(self, event: str) -> str:
return f"{self.service_name}_{event}"
@@ -364,9 +372,36 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
if self.platform_name == "Windows":
return _windows_process_identity(pid)
try:
return os.getpgid(pid)
process_group = os.getpgid(pid)
except OSError:
return None
started_at = self._posix_process_started_at(pid)
return f"{process_group}:{started_at}" if started_at else process_group
def _posix_process_started_at(self, pid: int) -> str | None:
if self.platform_name == "Linux":
try:
stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
except OSError:
return None
closing_paren = stat.rfind(")")
fields = stat[closing_paren + 2 :].split() if closing_paren >= 0 else []
# /proc/<pid>/stat fields after comm begin at field 3; starttime is field 22.
return fields[19] if len(fields) > 19 else None
if self.platform_name == "Darwin":
try:
result = self._subprocess_run(
["ps", "-o", "lstart=", "-p", str(pid)],
check=False,
capture_output=True,
text=True,
timeout=1,
)
except (OSError, subprocess.SubprocessError):
return None
started_at = getattr(result, "stdout", "").strip()
return started_at or None
return None
def _record_matches_process(self, state: dict[str, Any] | None, pid: int) -> bool:
if not state:
@@ -374,7 +409,13 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
recorded = state.get("identity")
if recorded is None:
return True
return recorded == self._process_identity(pid)
current = self._process_identity(pid)
if recorded == current:
return True
# Older POSIX state files stored only the process group id.
return isinstance(recorded, int) and isinstance(current, str) and current.startswith(
f"{recorded}:"
)
def _read_state(self) -> dict[str, Any] | None:
try: