deploy: channel-dependency gate + post-restart channel check
Test Suite / Detect changes (push) Canceled after 0s
Test Suite / Python (Windows, 3.14) (push) Canceled after 0s
Test Suite / Python (minimum, 3.11) (push) Canceled after 0s
Test Suite / Python (latest, 3.14 + coverage) (push) Canceled after 0s
Test Suite / webui (push) Canceled after 0s
Test Suite / Terminal UI (Windows) (push) Canceled after 0s
Test Suite / Terminal UI (push) Canceled after 0s
Test Suite / docker (push) Canceled after 0s
Test Suite / Detect changes (push) Canceled after 0s
Test Suite / Python (Windows, 3.14) (push) Canceled after 0s
Test Suite / Python (minimum, 3.11) (push) Canceled after 0s
Test Suite / Python (latest, 3.14 + coverage) (push) Canceled after 0s
Test Suite / webui (push) Canceled after 0s
Test Suite / Terminal UI (Windows) (push) Canceled after 0s
Test Suite / Terminal UI (push) Canceled after 0s
Test Suite / docker (push) Canceled after 0s
2026-09-03 incident: a silently-failed pip install (Tsinghua mirror transient) left the gateway WebSocket-only for 3.5 days while /health stayed green. New gates: - dep gate: verify venv imports (telegram, websockets) BEFORE restart; abort without touching the service on failure, one reinstall attempt Tsinghua -> PyPI (explicit package names; 'telegram' on PyPI is squat) - channel check: post-restart journal must have no 'Unknown channel' and 'Channels enabled:' must list every required channel, else auto-rollback (also applied on rollback path)
This commit is contained in:
+100
-2
@@ -16,6 +16,17 @@ STABILIZE_SEC=8
|
||||
HEALTH_TIMEOUT_SEC=30
|
||||
SERVICE="nanobot"
|
||||
|
||||
# Machine-specific channel contract (2026-09-03 incident): /health is
|
||||
# process-level and stays green when a configured channel silently fails to
|
||||
# load. Deploy-time gates below verify (1) channel dependencies are actually
|
||||
# importable in the venv BEFORE the service is touched, and (2) after restart,
|
||||
# every expected channel shows up in the startup "Channels enabled" line and
|
||||
# nothing logs "Unknown channel". Keep these in sync with config.json.
|
||||
REQUIRED_IMPORTS="telegram websockets" # python module names
|
||||
REQUIRED_PACKAGES="python-telegram-bot websockets" # matching PyPI names (module 'telegram' != package 'telegram'!)
|
||||
REQUIRED_CHANNELS="telegram websocket" # expected in 'Channels enabled:' startup line
|
||||
export XDG_RUNTIME_DIR=/run/user/1000
|
||||
|
||||
log() { echo "[deploy] $(date '+%H:%M:%S') $*"; }
|
||||
|
||||
# ---- helpers ----
|
||||
@@ -49,6 +60,73 @@ restart_service() {
|
||||
systemctl --user restart "$SERVICE"
|
||||
}
|
||||
|
||||
journal_since() { # $1 = since timestamp
|
||||
journalctl --user -u "$SERVICE" --since "$1" --output cat 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ---- channel-dependency gates (added after the 2026-09-03 silent outage) ----
|
||||
|
||||
deps_ok() {
|
||||
local py="$CHECKOUT/.venv/bin/python" mod
|
||||
[[ -x "$py" ]] || { log "ERROR: venv python missing at $py"; return 1; }
|
||||
for mod in $REQUIRED_IMPORTS; do
|
||||
"$py" -c "import $mod" 2>/dev/null || { log "dep gate: cannot import '$mod'"; return 1; }
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
verify_deps() {
|
||||
# Hard gate: never restart the service on a checkout whose channel
|
||||
# dependencies are missing. A failed mirror install must abort the deploy
|
||||
# (old process keeps running old code) — not silently degrade it.
|
||||
if deps_ok; then
|
||||
log "Dependency gate PASSED ($REQUIRED_IMPORTS)"
|
||||
return 0
|
||||
fi
|
||||
log "Dependency gate FAILED — attempting one reinstall (Tsinghua mirror -> PyPI)"
|
||||
"$CHECKOUT/.venv/bin/pip" install -q -i https://pypi.tuna.tsinghua.edu.cn/simple $REQUIRED_PACKAGES \
|
||||
|| "$CHECKOUT/.venv/bin/pip" install -q $REQUIRED_PACKAGES \
|
||||
|| true
|
||||
if deps_ok; then
|
||||
log "Dependency gate PASSED after reinstall"
|
||||
return 0
|
||||
fi
|
||||
log "ERROR: required channel deps still unimportable ($REQUIRED_PACKAGES)"
|
||||
return 1
|
||||
}
|
||||
|
||||
channels_check() { # $1 = since timestamp; 0 if all expected channels are up
|
||||
local since="$1" journal enabled ch
|
||||
journal=$(journal_since "$since")
|
||||
if grep -q "Unknown channel" <<<"$journal"; then
|
||||
log "CHANNEL CHECK FAILED: 'Unknown channel' logged since restart:"
|
||||
grep "Unknown channel" <<<"$journal" | head -3 | sed 's/^/[deploy] /'
|
||||
return 1
|
||||
fi
|
||||
enabled=$(grep -o "Channels enabled:.*" <<<"$journal" | tail -1 | sed 's/Channels enabled: *//' || true)
|
||||
if [[ -z "$enabled" ]]; then
|
||||
log "CHANNEL CHECK: no 'Channels enabled' startup line yet"
|
||||
return 1
|
||||
fi
|
||||
for ch in $REQUIRED_CHANNELS; do
|
||||
if ! grep -qw "$ch" <<<"$enabled"; then
|
||||
log "CHANNEL CHECK FAILED: expected '$ch', got 'Channels enabled: $enabled'"
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
log "Channel check PASSED ($enabled)"
|
||||
return 0
|
||||
}
|
||||
|
||||
channel_check_wait() { # $1 = since; retry up to ~30s for the startup line
|
||||
local since="$1" deadline=$((SECONDS + 30))
|
||||
while (( SECONDS < deadline )); do
|
||||
channels_check "$since" && return 0
|
||||
sleep 3
|
||||
done
|
||||
channels_check "$since"
|
||||
}
|
||||
|
||||
service_active() {
|
||||
export XDG_RUNTIME_DIR=/run/user/1000
|
||||
systemctl --user is-active --quiet "$SERVICE"
|
||||
@@ -107,8 +185,18 @@ cmd_deploy() {
|
||||
git checkout "$candidate_hash"
|
||||
fi
|
||||
|
||||
log "Verifying channel dependencies..."
|
||||
if ! verify_deps; then
|
||||
log "ERROR: aborting deploy — dependencies missing; service untouched"
|
||||
git checkout "$known_good"
|
||||
log "Checkout rolled back to known-good $(short_hash "$known_good")"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Restarting service..."
|
||||
restart_service
|
||||
local restart_ts
|
||||
restart_ts=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
log "Waiting ${STABILIZE_SEC}s for stabilization..."
|
||||
sleep "$STABILIZE_SEC"
|
||||
@@ -122,10 +210,18 @@ cmd_deploy() {
|
||||
log "Running health check..."
|
||||
if health_check; then
|
||||
log "Health check PASSED"
|
||||
else
|
||||
log "ERROR: health check FAILED"
|
||||
cmd_rollback_internal "$known_good" "$candidate_hash"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running channel check..."
|
||||
if channel_check_wait "$restart_ts"; then
|
||||
write_state "$candidate_hash"
|
||||
log "Promoted $(short_hash "$candidate_hash") to known-good"
|
||||
else
|
||||
log "ERROR: health check FAILED"
|
||||
log "ERROR: channel check FAILED"
|
||||
cmd_rollback_internal "$known_good" "$candidate_hash"
|
||||
return 1
|
||||
fi
|
||||
@@ -140,10 +236,12 @@ cmd_rollback_internal() {
|
||||
cd "$CHECKOUT"
|
||||
git checkout "$target"
|
||||
restart_service
|
||||
local restart_ts
|
||||
restart_ts=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
sleep "$STABILIZE_SEC"
|
||||
|
||||
if service_active && health_check; then
|
||||
if service_active && health_check && channel_check_wait "$restart_ts"; then
|
||||
log "Rollback successful, service healthy"
|
||||
# Write failure report
|
||||
cat > "/home/claw/.nanobot/deploy-failure.json" <<EOF
|
||||
|
||||
Reference in New Issue
Block a user