From d8fc220c8bfaad97fb910a301ca00c4310bbe05e Mon Sep 17 00:00:00 2001 From: nanobot Date: Sun, 6 Sep 2026 22:16:03 +0800 Subject: [PATCH] deploy: channel-dependency gate + post-restart channel check 2026-09-03 incident: a silently-failed pip install (Tsinghua mirror transient) left the gateway WebSocket-only for 3.5 days while /health stayed green. New gates: - dep gate: verify venv imports (telegram, websockets) BEFORE restart; abort without touching the service on failure, one reinstall attempt Tsinghua -> PyPI (explicit package names; 'telegram' on PyPI is squat) - channel check: post-restart journal must have no 'Unknown channel' and 'Channels enabled:' must list every required channel, else auto-rollback (also applied on rollback path) --- scripts/deploy.sh | 104 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index be0b65b0..faf2864a 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -16,6 +16,17 @@ STABILIZE_SEC=8 HEALTH_TIMEOUT_SEC=30 SERVICE="nanobot" +# Machine-specific channel contract (2026-09-03 incident): /health is +# process-level and stays green when a configured channel silently fails to +# load. Deploy-time gates below verify (1) channel dependencies are actually +# importable in the venv BEFORE the service is touched, and (2) after restart, +# every expected channel shows up in the startup "Channels enabled" line and +# nothing logs "Unknown channel". Keep these in sync with config.json. +REQUIRED_IMPORTS="telegram websockets" # python module names +REQUIRED_PACKAGES="python-telegram-bot websockets" # matching PyPI names (module 'telegram' != package 'telegram'!) +REQUIRED_CHANNELS="telegram websocket" # expected in 'Channels enabled:' startup line +export XDG_RUNTIME_DIR=/run/user/1000 + log() { echo "[deploy] $(date '+%H:%M:%S') $*"; } # ---- helpers ---- @@ -49,6 +60,73 @@ restart_service() { systemctl --user restart "$SERVICE" } +journal_since() { # $1 = since timestamp + journalctl --user -u "$SERVICE" --since "$1" --output cat 2>/dev/null || true +} + +# ---- channel-dependency gates (added after the 2026-09-03 silent outage) ---- + +deps_ok() { + local py="$CHECKOUT/.venv/bin/python" mod + [[ -x "$py" ]] || { log "ERROR: venv python missing at $py"; return 1; } + for mod in $REQUIRED_IMPORTS; do + "$py" -c "import $mod" 2>/dev/null || { log "dep gate: cannot import '$mod'"; return 1; } + done + return 0 +} + +verify_deps() { + # Hard gate: never restart the service on a checkout whose channel + # dependencies are missing. A failed mirror install must abort the deploy + # (old process keeps running old code) — not silently degrade it. + if deps_ok; then + log "Dependency gate PASSED ($REQUIRED_IMPORTS)" + return 0 + fi + log "Dependency gate FAILED — attempting one reinstall (Tsinghua mirror -> PyPI)" + "$CHECKOUT/.venv/bin/pip" install -q -i https://pypi.tuna.tsinghua.edu.cn/simple $REQUIRED_PACKAGES \ + || "$CHECKOUT/.venv/bin/pip" install -q $REQUIRED_PACKAGES \ + || true + if deps_ok; then + log "Dependency gate PASSED after reinstall" + return 0 + fi + log "ERROR: required channel deps still unimportable ($REQUIRED_PACKAGES)" + return 1 +} + +channels_check() { # $1 = since timestamp; 0 if all expected channels are up + local since="$1" journal enabled ch + journal=$(journal_since "$since") + if grep -q "Unknown channel" <<<"$journal"; then + log "CHANNEL CHECK FAILED: 'Unknown channel' logged since restart:" + grep "Unknown channel" <<<"$journal" | head -3 | sed 's/^/[deploy] /' + return 1 + fi + enabled=$(grep -o "Channels enabled:.*" <<<"$journal" | tail -1 | sed 's/Channels enabled: *//' || true) + if [[ -z "$enabled" ]]; then + log "CHANNEL CHECK: no 'Channels enabled' startup line yet" + return 1 + fi + for ch in $REQUIRED_CHANNELS; do + if ! grep -qw "$ch" <<<"$enabled"; then + log "CHANNEL CHECK FAILED: expected '$ch', got 'Channels enabled: $enabled'" + return 1 + fi + done + log "Channel check PASSED ($enabled)" + return 0 +} + +channel_check_wait() { # $1 = since; retry up to ~30s for the startup line + local since="$1" deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + channels_check "$since" && return 0 + sleep 3 + done + channels_check "$since" +} + service_active() { export XDG_RUNTIME_DIR=/run/user/1000 systemctl --user is-active --quiet "$SERVICE" @@ -107,8 +185,18 @@ cmd_deploy() { git checkout "$candidate_hash" fi + log "Verifying channel dependencies..." + if ! verify_deps; then + log "ERROR: aborting deploy — dependencies missing; service untouched" + git checkout "$known_good" + log "Checkout rolled back to known-good $(short_hash "$known_good")" + return 1 + fi + log "Restarting service..." restart_service + local restart_ts + restart_ts=$(date '+%Y-%m-%d %H:%M:%S') log "Waiting ${STABILIZE_SEC}s for stabilization..." sleep "$STABILIZE_SEC" @@ -122,10 +210,18 @@ cmd_deploy() { log "Running health check..." if health_check; then log "Health check PASSED" + else + log "ERROR: health check FAILED" + cmd_rollback_internal "$known_good" "$candidate_hash" + return 1 + fi + + log "Running channel check..." + if channel_check_wait "$restart_ts"; then write_state "$candidate_hash" log "Promoted $(short_hash "$candidate_hash") to known-good" else - log "ERROR: health check FAILED" + log "ERROR: channel check FAILED" cmd_rollback_internal "$known_good" "$candidate_hash" return 1 fi @@ -140,10 +236,12 @@ cmd_rollback_internal() { cd "$CHECKOUT" git checkout "$target" restart_service + local restart_ts + restart_ts=$(date '+%Y-%m-%d %H:%M:%S') sleep "$STABILIZE_SEC" - if service_active && health_check; then + if service_active && health_check && channel_check_wait "$restart_ts"; then log "Rollback successful, service healthy" # Write failure report cat > "/home/claw/.nanobot/deploy-failure.json" <