fix: use dedicated _shutting_down flag for cancel-time checkpoint restore

The previous guard used self._running, which is False both before run()
starts and during shutdown. Tests and process_direct() dispatch messages
with _running=False, so cancelled turns there silently skipped the
checkpoint restore and lost partial context. Introduce _shutting_down,
set only by stop(), and gate the restore on it.

DEPLOYING-NIXOS.md

@@ -0,0 +1,194 @@
+# Deploying nanobot on NixOS
+
+This documents what it took to get nanobot running on Camellia (NixOS 26.05).
+
+See `/etc/nixos/configuration.nix` for the full config — every block below has a matching section there.
+
+## 1. Base system
+
+```nix
+# Hostname, timezone, SSH, fail2ban
+networking.hostName = "Camellia";
+time.timeZone = "Asia/Shanghai";
+services.openssh = { enable = true; ... };
+services.fail2ban = { enable = true; };
+```
+
+## 2. Required packages
+
+```nix
+environment.systemPackages = with pkgs; [
+  git curl wget vim tmux htop jq fd ripgrep
+  python313 uv     # nanobot runtime
+  nodejs_22        # for some skills/tools
+];
+```
+
+## 3. Users
+
+Create a dedicated user for nanobot with lingering enabled:
+
+```nix
+users.users.claw = {
+  extraGroups = ["sudo"];
+  isNormalUser = true;
+  linger = true;   # user services survive logout
+};
+security.sudo.extraRules = [
+  { users = ["claw"]; commands = [{ command = "ALL"; options = ["NOPASSWD"]; }]; }
+];
+```
+
+## 4. FHS symlinks
+
+nanobot's `exec` tool (and many scripts) expect `/bin/bash`:
+
+```nix
+systemd.tmpfiles.rules = [
+  "L+ /bin/bash - - - - /run/current-system/sw/bin/bash"
+  "L+ /usr/bin/env - - - - /run/current-system/sw/bin/env"
+];
+```
+
+## 5. nanobot systemd user service
+
+```nix
+systemd.user.services.nanobot = {
+  description = "nanobot AI assistant gateway";
+  wantedBy = ["default.target"];
+  serviceConfig = {
+    Type = "simple";
+    ExecStart = "/home/claw/nanobot-src/.venv/bin/python -m nanobot gateway --foreground --port 18790";
+    Restart = "always";
+    RestartSec = 5;
+    WorkingDirectory = "/home/claw/.nanobot/workspace";
+    Environment = "HOME=/home/claw";
+  };
+};
+```
+
+Use a git checkout (`/home/claw/nanobot-src`) so you can `git checkout known-good` to rollback.
+
+## 6. Watchdog timer
+
+Covers the case where systemd misses a crash:
+
+```nix
+systemd.user.services.nanobot-watchdog = {
+  serviceConfig = {
+    Type = "oneshot";
+    ExecStart = "${pkgs.systemd}/bin/systemctl --user start nanobot";
+  };
+};
+systemd.user.timers.nanobot-watchdog = {
+  timerConfig = { OnUnitActiveSec = "60s"; };
+};
+```
+
+## 7. Reverse proxy (Caddy)
+
+nanobot's gateway binds to `127.0.0.1` by default and rejects non-local connections. For a dedicated VM this is over-cautious, so we use Caddy to reverse-proxy the WebSocket UI externally:
+
+```nix
+services.caddy = {
+  enable = true;
+  virtualHosts."http://192.168.122.194".extraConfig = ''
+    reverse_proxy 127.0.0.1:8765
+  '';
+};
+```
+
+## 8. Proxy for outbound traffic (v2rayA)
+
+If the LLM API and Telegram are blocked:
+
+```nix
+services.v2raya = { enable = true; };
+```
+
+Then configure proxy env vars in `~/.bashrc`:
+
+```bash
+export http_proxy=http://127.0.0.1:20171
+export https_proxy=http://127.0.0.1:20171
+export all_proxy=socks5://127.0.0.1:20170
+```
+
+## 9. OOM protection
+
+nanobot can spike memory. Without this, the kernel OOM killer takes it down:
+
+```nix
+services.earlyoom = {
+  enable = true;
+  freeMemThreshold = 5;
+  freeSwapThreshold = 10;
+};
+swapDevices = [{ device = "/swapfile"; size = 2048; }];  # 2 GB
+```
+
+## 10. Firewall
+
+```nix
+networking.firewall.allowedTCPPorts = [
+  22     # SSH
+  80     # Caddy HTTP
+  2017   # v2rayA web UI
+  20170  # v2rayA SOCKS5
+  20171  # v2rayA HTTP proxy
+];
+```
+
+## 11. Kernel hardening
+
+```nix
+boot.kernel.sysctl = {
+  "net.ipv4.tcp_syncookies" = 1;
+  "net.ipv4.conf.all.rp_filter" = 1;
+  "net.ipv4.conf.all.accept_redirects" = 0;
+  "net.ipv6.conf.all.accept_redirects" = 0;
+  "kernel.kptr_restrict" = 1;
+  "kernel.dmesg_restrict" = 1;
+};
+```
+
+## 12. Nix GC
+
+```nix
+nix.gc = { automatic = true; dates = "weekly"; options = "--delete-older-than 30d"; };
+```
+
+## 13. Mirrors (for China)
+
+```nix
+nix.settings.substituters = ["https://mirrors.tuna.tsinghua.edu.cn/nix-channels/store"];
+```
+
+## 14. Install nanobot itself
+
+```bash
+# As claw user
+git clone https://github.com/HKUDS/nanobot.git ~/nanobot-src
+cd ~/nanobot-src
+uv sync
+uv run nanobot onboard   # interactive setup wizard
+# Populate workspace: SOUL.md, USER.md, memory/MEMORY.md
+```
+
+## 15. Apply
+
+Use the safe-switch wrapper to rebuild with health checks:
+
+```bash
+sudo /home/claw/.nanobot/workspace/scripts/safe-switch.sh
+```
+
+This runs `nixos-rebuild switch`, waits, then verifies the gateway is reachable.
+If the health check fails, it automatically rolls back.
+
+## Known quirks
+
+- **sudo path**: Use `/run/wrappers/bin/sudo` inside scripts, not the default PATH's `/run/current-system/sw/bin/sudo` (lacks setuid).
+- **systemctl from cron/timers**: Need `XDG_RUNTIME_DIR=/run/user/1000 DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus`.
+- **Gateway self-restart**: Never `systemctl --user restart nanobot` from within nanobot's own process — it kills the parent. Use `nohup bash -c 'sleep 2 && ...' &`.
+- **Rollback**: `cd ~/nanobot-src && git checkout known-good && systemctl --user restart nanobot`.

nanobot/agent/loop.py

@@ -74,7 +74,11 @@ from nanobot.session.goal_state import (
     sustained_goal_active,
 )
 from nanobot.session.history_visibility import HIDDEN_HISTORY_META
-from nanobot.session.keys import UNIFIED_SESSION_KEY, remember_last_channel
+from nanobot.session.keys import (
+    UNIFIED_SESSION_KEY,
+    last_channel_from_metadata,
+    remember_last_channel,
+)
 from nanobot.session.manager import (
     Session,
     SessionManager,
@@ -374,6 +378,7 @@ class AgentLoop:
         )
         self._unified_session = unified_session
         self._running = False
+        self._shutting_down = False
         self._mcp_servers = mcp_servers or {}
         self._mcp_stacks: dict[str, MCPConnection] = {}
         self._mcp_connecting = False
@@ -1219,22 +1224,28 @@ class AgentLoop:
                     # _emit_checkpoint during tool execution; materializing
                     # it into session history now makes it visible in the
                     # next conversation turn.
-                    try:
+                    #
-                        key = self._effective_session_key(msg)
+                    # During gateway shutdown (self._shutting_down is True),
-                        session = self.sessions.get_or_create(key)
+                    # skip this so the pending markers survive and
-                        if self._restore_runtime_checkpoint(session):
+                    # recover_stale_sessions() can notify the user and
-                            self._clear_pending_user_turn(session)
+                    # re-trigger the interrupted turn on next startup.
-                            self.sessions.save(session)
+                    if not self._shutting_down:
-                            logger.info(
+                        try:
-                                "Restored partial context for cancelled session {}",
+                            key = self._effective_session_key(msg)
-                                key,
+                            session = self.sessions.get_or_create(key)
+                            if self._restore_runtime_checkpoint(session):
+                                self._clear_pending_user_turn(session)
+                                self.sessions.save(session)
+                                logger.info(
+                                    "Restored partial context for cancelled session {}",
+                                    key,
+                                )
+                        except Exception:
+                            logger.debug(
+                                "Could not restore checkpoint for cancelled session {}",
+                                session_key,
+                                exc_info=True,
                             )
-                    except Exception:
-                        logger.debug(
-                            "Could not restore checkpoint for cancelled session {}",
-                            session_key,
-                            exc_info=True,
-                        )
                     raise
                 except Exception as exc:
                     logger.exception("Error processing message for session {}", session_key)
@@ -1308,6 +1319,7 @@ class AgentLoop:
     def stop(self) -> None:
         """Stop the agent loop."""
         self._running = False
+        self._shutting_down = True
         logger.info("Agent loop stopping")
 
     async def _process_message(
@@ -2042,6 +2054,10 @@ class AgentLoop:
     def recover_stale_sessions(self) -> int:
         """Scan all sessions on startup and recover any with stale turn state.
 
+        For each recovered session, publishes a restart notification to the
+        user and re-triggers any interrupted turn so work continues
+        automatically after a crash or restart.
+
         Returns the number of sessions that were recovered.
         """
         recovered = 0
@@ -2049,10 +2065,17 @@ class AgentLoop:
             try:
                 session = self.sessions.get_or_create(key)
                 changed = False
+                # Capture pending flag before _restore_runtime_checkpoint
+                # clears it (it clears both checkpoint and pending_turn at
+                # lines 1966-1967).
+                had_pending = bool(
+                    session.metadata.get(self._PENDING_USER_TURN_KEY)
+                )
                 if self._restore_runtime_checkpoint(session):
                     changed = True
                 if self._restore_pending_user_turn(session):
                     changed = True
+                    had_pending = True
                 if changed:
                     self.sessions.save(session)
                     recovered += 1
@@ -2060,6 +2083,8 @@ class AgentLoop:
                         "Recovered stale session {} on startup",
                         key,
                     )
+                    # Notify the user and re-trigger interrupted work.
+                    self._notify_session_recovered(key, session, had_pending)
             except Exception:
                 logger.debug(
                     "Could not recover stale session {}",
@@ -2069,3 +2094,80 @@ class AgentLoop:
         if recovered:
             logger.info("Recovered {} stale session(s) on startup", recovered)
         return recovered
+
+    def _notify_session_recovered(
+        self,
+        session_key: str,
+        session: Session,
+        had_pending_turn: bool,
+    ) -> None:
+        """Publish restart notification and re-trigger interrupted turns.
+
+        Uses the bus queues directly so messages are delivered once the
+        outbound dispatcher and agent loop start.
+        """
+        route = last_channel_from_metadata(session.metadata)
+        if route is None:
+            # Fall back to parsing the session key (format: channel:chat_id).
+            if ":" in session_key and not session_key.startswith("unified:"):
+                parts = session_key.split(":", 1)
+                if parts[0] and parts[1]:
+                    route = (parts[0], parts[1])
+        if route is None:
+            logger.debug(
+                "Cannot determine route for recovered session {}",
+                session_key,
+            )
+            return
+        channel, chat_id = route
+        # Notify the user that a restart happened and the session was recovered.
+        try:
+            self.bus.outbound.put_nowait(
+                OutboundMessage(
+                    channel=channel,
+                    chat_id=chat_id,
+                    content=(
+                        "🔄 nanobot was restarted (crash or deploy). "
+                        "Your session has been recovered."
+                    ),
+                )
+            )
+        except Exception:
+            logger.debug(
+                "Could not enqueue restart notification for {}",
+                session_key,
+                exc_info=True,
+            )
+        # Re-trigger the interrupted turn so work continues automatically.
+        if had_pending_turn:
+            last_user_msg = None
+            for msg in reversed(session.messages):
+                if msg.get("role") == "user":
+                    last_user_msg = msg
+                    break
+            if last_user_msg and last_user_msg.get("content"):
+                try:
+                    self.bus.inbound.put_nowait(
+                        InboundMessage(
+                            channel=channel,
+                            sender_id=chat_id,
+                            chat_id=chat_id,
+                            content=last_user_msg["content"],
+                            # The message already exists in session history
+                            # (persisted before the crash); skip persisting it
+                            # again so recovery does not duplicate it.
+                            metadata={
+                                turn_continuation.SKIP_USER_PERSIST_META: True
+                            },
+                        )
+                    )
+                    logger.info(
+                        "Re-published user message for recovered session {}",
+                        session_key,
+                    )
+                except Exception:
+                    logger.debug(
+                        "Could not re-publish message for {}",
+                        session_key,
+                        exc_info=True,
+                    )

nanobot/channels/telegram/runtime.py

@@ -729,7 +729,7 @@ class TelegramChannel(BaseChannel):
         """Send a message through Telegram."""
         if not self._app:
             self.logger.warning("bot not running")
-            return
+            raise RuntimeError("Telegram bot is not running")
 
         progress_event = msg.event if isinstance(msg.event, ProgressEvent) else None