deploy: channel-dependency gate + post-restart channel check
Test Suite / Detect changes (push) Canceled after 0s
Test Suite / Python (Windows, 3.14) (push) Canceled after 0s
Test Suite / Python (minimum, 3.11) (push) Canceled after 0s
Test Suite / Python (latest, 3.14 + coverage) (push) Canceled after 0s
Test Suite / webui (push) Canceled after 0s
Test Suite / Terminal UI (Windows) (push) Canceled after 0s
Test Suite / Terminal UI (push) Canceled after 0s
Test Suite / docker (push) Canceled after 0s

2026-09-03 incident: a silently-failed pip install (Tsinghua mirror
transient) left the gateway WebSocket-only for 3.5 days while /health
stayed green. New gates:
- dep gate: verify venv imports (telegram, websockets) BEFORE restart;
  abort without touching the service on failure, one reinstall attempt
  Tsinghua -> PyPI (explicit package names; 'telegram' on PyPI is squat)
- channel check: post-restart journal must have no 'Unknown channel'
  and 'Channels enabled:' must list every required channel, else
  auto-rollback (also applied on rollback path)
This commit is contained in:
nanobot
2026-09-06 22:16:03 +08:00
parent aabad78df9
commit d8fc220c8b
+100 -2
View File
@@ -16,6 +16,17 @@ STABILIZE_SEC=8
HEALTH_TIMEOUT_SEC=30 HEALTH_TIMEOUT_SEC=30
SERVICE="nanobot" SERVICE="nanobot"
# Machine-specific channel contract (2026-09-03 incident): /health is
# process-level and stays green when a configured channel silently fails to
# load. Deploy-time gates below verify (1) channel dependencies are actually
# importable in the venv BEFORE the service is touched, and (2) after restart,
# every expected channel shows up in the startup "Channels enabled" line and
# nothing logs "Unknown channel". Keep these in sync with config.json.
REQUIRED_IMPORTS="telegram websockets" # python module names
REQUIRED_PACKAGES="python-telegram-bot websockets" # matching PyPI names (module 'telegram' != package 'telegram'!)
REQUIRED_CHANNELS="telegram websocket" # expected in 'Channels enabled:' startup line
export XDG_RUNTIME_DIR=/run/user/1000
log() { echo "[deploy] $(date '+%H:%M:%S') $*"; } log() { echo "[deploy] $(date '+%H:%M:%S') $*"; }
# ---- helpers ---- # ---- helpers ----
@@ -49,6 +60,73 @@ restart_service() {
systemctl --user restart "$SERVICE" systemctl --user restart "$SERVICE"
} }
journal_since() { # $1 = since timestamp
journalctl --user -u "$SERVICE" --since "$1" --output cat 2>/dev/null || true
}
# ---- channel-dependency gates (added after the 2026-09-03 silent outage) ----
deps_ok() {
local py="$CHECKOUT/.venv/bin/python" mod
[[ -x "$py" ]] || { log "ERROR: venv python missing at $py"; return 1; }
for mod in $REQUIRED_IMPORTS; do
"$py" -c "import $mod" 2>/dev/null || { log "dep gate: cannot import '$mod'"; return 1; }
done
return 0
}
verify_deps() {
# Hard gate: never restart the service on a checkout whose channel
# dependencies are missing. A failed mirror install must abort the deploy
# (old process keeps running old code) — not silently degrade it.
if deps_ok; then
log "Dependency gate PASSED ($REQUIRED_IMPORTS)"
return 0
fi
log "Dependency gate FAILED — attempting one reinstall (Tsinghua mirror -> PyPI)"
"$CHECKOUT/.venv/bin/pip" install -q -i https://pypi.tuna.tsinghua.edu.cn/simple $REQUIRED_PACKAGES \
|| "$CHECKOUT/.venv/bin/pip" install -q $REQUIRED_PACKAGES \
|| true
if deps_ok; then
log "Dependency gate PASSED after reinstall"
return 0
fi
log "ERROR: required channel deps still unimportable ($REQUIRED_PACKAGES)"
return 1
}
channels_check() { # $1 = since timestamp; 0 if all expected channels are up
local since="$1" journal enabled ch
journal=$(journal_since "$since")
if grep -q "Unknown channel" <<<"$journal"; then
log "CHANNEL CHECK FAILED: 'Unknown channel' logged since restart:"
grep "Unknown channel" <<<"$journal" | head -3 | sed 's/^/[deploy] /'
return 1
fi
enabled=$(grep -o "Channels enabled:.*" <<<"$journal" | tail -1 | sed 's/Channels enabled: *//' || true)
if [[ -z "$enabled" ]]; then
log "CHANNEL CHECK: no 'Channels enabled' startup line yet"
return 1
fi
for ch in $REQUIRED_CHANNELS; do
if ! grep -qw "$ch" <<<"$enabled"; then
log "CHANNEL CHECK FAILED: expected '$ch', got 'Channels enabled: $enabled'"
return 1
fi
done
log "Channel check PASSED ($enabled)"
return 0
}
channel_check_wait() { # $1 = since; retry up to ~30s for the startup line
local since="$1" deadline=$((SECONDS + 30))
while (( SECONDS < deadline )); do
channels_check "$since" && return 0
sleep 3
done
channels_check "$since"
}
service_active() { service_active() {
export XDG_RUNTIME_DIR=/run/user/1000 export XDG_RUNTIME_DIR=/run/user/1000
systemctl --user is-active --quiet "$SERVICE" systemctl --user is-active --quiet "$SERVICE"
@@ -107,8 +185,18 @@ cmd_deploy() {
git checkout "$candidate_hash" git checkout "$candidate_hash"
fi fi
log "Verifying channel dependencies..."
if ! verify_deps; then
log "ERROR: aborting deploy — dependencies missing; service untouched"
git checkout "$known_good"
log "Checkout rolled back to known-good $(short_hash "$known_good")"
return 1
fi
log "Restarting service..." log "Restarting service..."
restart_service restart_service
local restart_ts
restart_ts=$(date '+%Y-%m-%d %H:%M:%S')
log "Waiting ${STABILIZE_SEC}s for stabilization..." log "Waiting ${STABILIZE_SEC}s for stabilization..."
sleep "$STABILIZE_SEC" sleep "$STABILIZE_SEC"
@@ -122,10 +210,18 @@ cmd_deploy() {
log "Running health check..." log "Running health check..."
if health_check; then if health_check; then
log "Health check PASSED" log "Health check PASSED"
else
log "ERROR: health check FAILED"
cmd_rollback_internal "$known_good" "$candidate_hash"
return 1
fi
log "Running channel check..."
if channel_check_wait "$restart_ts"; then
write_state "$candidate_hash" write_state "$candidate_hash"
log "Promoted $(short_hash "$candidate_hash") to known-good" log "Promoted $(short_hash "$candidate_hash") to known-good"
else else
log "ERROR: health check FAILED" log "ERROR: channel check FAILED"
cmd_rollback_internal "$known_good" "$candidate_hash" cmd_rollback_internal "$known_good" "$candidate_hash"
return 1 return 1
fi fi
@@ -140,10 +236,12 @@ cmd_rollback_internal() {
cd "$CHECKOUT" cd "$CHECKOUT"
git checkout "$target" git checkout "$target"
restart_service restart_service
local restart_ts
restart_ts=$(date '+%Y-%m-%d %H:%M:%S')
sleep "$STABILIZE_SEC" sleep "$STABILIZE_SEC"
if service_active && health_check; then if service_active && health_check && channel_check_wait "$restart_ts"; then
log "Rollback successful, service healthy" log "Rollback successful, service healthy"
# Write failure report # Write failure report
cat > "/home/claw/.nanobot/deploy-failure.json" <<EOF cat > "/home/claw/.nanobot/deploy-failure.json" <<EOF