fix(shell): reap zombie processes on all subprocess exit paths
The previous fix (dbcc7cb5) only added os.waitpid() to _kill_process(),
covering the timeout/cancel path of one-shot exec. Zombies continued to
accumulate because several other exit paths never reaped children:
- _ExecSession.kill(): sent SIGKILL + process.wait(5s) but had no
os.waitpid() fallback if the wait timed out
- ExecTool.execute() generic exception handler: leaked the subprocess
if communicate() raised an unexpected error
- Normal completion paths: relied entirely on asyncio's child-watcher,
which can miss exits inside Docker containers (pidfd/SIGCHLD gaps)
Changes:
- Extract _reap_pid() helper for consistent, safe os.waitpid(WNOHANG)
- Add _reap_pid() fallback to _ExecSession.kill() via try/finally
- Add _reap_pid() safety-net after normal process exit in both
ExecTool.execute() and _ExecSession.poll()
- Kill + reap subprocess in the generic except Exception handler
- Add periodic zombie reaper background task (every 30s) in the
gateway as a last line of defense
This commit is contained in:
@@ -201,6 +201,7 @@ app = typer.Typer(
|
||||
)
|
||||
|
||||
console = Console()
|
||||
_IS_WINDOWS = sys.platform == "win32"
|
||||
EXIT_COMMANDS = {"exit", "quit", "/exit", "/quit", ":q"}
|
||||
_REASONING_SENTENCE_ENDINGS = (".", "!", "?", "。", "!", "?")
|
||||
_REASONING_FLUSH_CHARS = 60
|
||||
@@ -1695,6 +1696,29 @@ def _run_gateway(
|
||||
else:
|
||||
console.print("[yellow]✗[/yellow] Heartbeat: disabled")
|
||||
|
||||
async def _zombie_reaper() -> None:
|
||||
"""Periodically reap zombie child processes.
|
||||
|
||||
asyncio's child-watcher *should* reap all children, but inside
|
||||
Docker containers the pidfd / SIGCHLD mechanism can miss exits.
|
||||
This task runs every 30 s and calls ``os.waitpid(-1, WNOHANG)``
|
||||
in a loop to collect any zombies that slipped through.
|
||||
"""
|
||||
_INTERVAL = 30
|
||||
while True:
|
||||
await asyncio.sleep(_INTERVAL)
|
||||
reaped = 0
|
||||
while True:
|
||||
try:
|
||||
pid, _ = os.waitpid(-1, os.WNOHANG)
|
||||
if pid == 0:
|
||||
break # no more zombie children
|
||||
reaped += 1
|
||||
except ChildProcessError:
|
||||
break # no child processes at all
|
||||
if reaped:
|
||||
logger.info("Zombie reaper: reaped {} defunct child process(es)", reaped)
|
||||
|
||||
async def _health_server(host: str, health_port: int):
|
||||
"""Lightweight HTTP health endpoint on the gateway port."""
|
||||
import json as _json
|
||||
@@ -1820,6 +1844,11 @@ def _run_gateway(
|
||||
name="nanobot-local-triggers",
|
||||
),
|
||||
]
|
||||
if not _IS_WINDOWS:
|
||||
tasks.append(asyncio.create_task(
|
||||
_zombie_reaper(),
|
||||
name="nanobot-zombie-reaper",
|
||||
))
|
||||
if health_server_enabled:
|
||||
tasks.append(asyncio.create_task(
|
||||
_health_server(config.gateway.host, port),
|
||||
|
||||
Reference in New Issue
Block a user