nanobot/scripts/deploy.sh

#!/usr/bin/env bash
# nanobot deploy/rollback supervisor
# Manages git-checkout-based deployments with health checks and automatic rollback.
#
# Usage:
#   deploy.sh <commit-ish>     Deploy a candidate commit
#   deploy.sh rollback          Rollback to last known-good
#   deploy.sh status            Show current deployment state

set -euo pipefail

CHECKOUT="/home/claw/nanobot-src"
STATE_FILE="/home/claw/.nanobot/deploy-state.json"
HEALTH_URL="http://127.0.0.1:18790/health"
STABILIZE_SEC=8
HEALTH_TIMEOUT_SEC=30
SERVICE="nanobot"

# Machine-specific channel contract (2026-09-03 incident): /health is
# process-level and stays green when a configured channel silently fails to
# load. Deploy-time gates below verify (1) channel dependencies are actually
# importable in the venv BEFORE the service is touched, and (2) after restart,
# every expected channel shows up in the startup "Channels enabled" line and
# nothing logs "Unknown channel". Keep these in sync with config.json.
REQUIRED_IMPORTS="telegram websockets nio"        # python module names (nio = matrix-nio)
REQUIRED_PACKAGES="python-telegram-bot websockets matrix-nio"  # matching PyPI names (module 'telegram' != package 'telegram'!)
REQUIRED_CHANNELS="telegram websocket matrix"     # expected in 'Channels enabled:' startup line
export XDG_RUNTIME_DIR=/run/user/1000

log() { echo "[deploy] $(date '+%H:%M:%S') $*"; }

# ---- helpers ----

current_commit() { git -C "$CHECKOUT" rev-parse HEAD; }
short_hash() { echo "${1:0:8}"; }

read_state() {
    if [[ -f "$STATE_FILE" ]]; then
        python3 -c "import json; d=json.load(open('$STATE_FILE')); print(d.get('known_good',''))"
    else
        echo ""
    fi
}

write_state() {
    local known_good="$1"
    python3 -c "
import json, sys
d = {}
try: d = json.load(open('$STATE_FILE'))
except: pass
d['known_good'] = '$known_good'
d['last_updated'] = '$(date -Iseconds)'
json.dump(d, open('$STATE_FILE', 'w'), indent=2)
"
}

restart_service() {
    export XDG_RUNTIME_DIR=/run/user/1000
    systemctl --user restart "$SERVICE"
}

journal_since() {  # $1 = since timestamp
    journalctl --user -u "$SERVICE" --since "$1" --output cat 2>/dev/null || true
}

# ---- channel-dependency gates (added after the 2026-09-03 silent outage) ----

deps_ok() {
    local py="$CHECKOUT/.venv/bin/python" mod
    [[ -x "$py" ]] || { log "ERROR: venv python missing at $py"; return 1; }
    for mod in $REQUIRED_IMPORTS; do
        "$py" -c "import $mod" 2>/dev/null || { log "dep gate: cannot import '$mod'"; return 1; }
    done
    return 0
}

verify_deps() {
    # Hard gate: never restart the service on a checkout whose channel
    # dependencies are missing. A failed mirror install must abort the deploy
    # (old process keeps running old code) — not silently degrade it.
    if deps_ok; then
        log "Dependency gate PASSED ($REQUIRED_IMPORTS)"
        return 0
    fi
    log "Dependency gate FAILED — attempting one reinstall (Tsinghua mirror -> PyPI)"
    "$CHECKOUT/.venv/bin/pip" install -q -i https://pypi.tuna.tsinghua.edu.cn/simple $REQUIRED_PACKAGES \
        || "$CHECKOUT/.venv/bin/pip" install -q $REQUIRED_PACKAGES \
        || true
    if deps_ok; then
        log "Dependency gate PASSED after reinstall"
        return 0
    fi
    log "ERROR: required channel deps still unimportable ($REQUIRED_PACKAGES)"
    return 1
}

channels_check() {  # $1 = since timestamp; 0 if all expected channels are up
    local since="$1" journal enabled ch
    journal=$(journal_since "$since")
    # Match the actual warning-pipeline format, not bare 'Unknown channel' —
    # the gateway also logs agent tool calls to this journal, and those echo
    # the literal string when the agent greps for it (false positives).
    if grep -qF '| WARNING | - | Unknown channel:' <<<"$journal"; then
        log "CHANNEL CHECK FAILED: 'Unknown channel' logged since restart:"
        grep -F '| WARNING | - | Unknown channel:' <<<"$journal" | head -3 | sed 's/^/[deploy]     /'
        return 1
    fi
    enabled=$(grep -o "Channels enabled:.*" <<<"$journal" | tail -1 | sed 's/Channels enabled: *//' || true)
    if [[ -z "$enabled" ]]; then
        log "CHANNEL CHECK: no 'Channels enabled' startup line yet"
        return 1
    fi
    for ch in $REQUIRED_CHANNELS; do
        if ! grep -qw "$ch" <<<"$enabled"; then
            log "CHANNEL CHECK FAILED: expected '$ch', got 'Channels enabled: $enabled'"
            return 1
        fi
    done
    log "Channel check PASSED ($enabled)"
    return 0
}

channel_check_wait() {  # $1 = since; retry up to ~30s for the startup line
    local since="$1" deadline=$((SECONDS + 30))
    while (( SECONDS < deadline )); do
        channels_check "$since" && return 0
        sleep 3
    done
    channels_check "$since"
}

service_active() {
    export XDG_RUNTIME_DIR=/run/user/1000
    systemctl --user is-active --quiet "$SERVICE"
}

health_check() {
    # Returns 0 if healthy, 1 if not
    local deadline=$((SECONDS + HEALTH_TIMEOUT_SEC))
    while (( SECONDS < deadline )); do
        if curl -sf --noproxy '*' --max-time 3 "$HEALTH_URL" >/dev/null 2>&1; then
            return 0
        fi
        sleep 1
    done
    return 1
}

# ---- commands ----

cmd_status() {
    echo "Checkout: $CHECKOUT"
    echo "Current commit: $(current_commit) ($(short_hash "$(current_commit)"))"
    echo "Known-good:     $(read_state)"
    echo "Service:        $(service_active && echo 'active' || echo 'inactive')"
}

cmd_deploy() {
    local candidate="$1"
    local known_good
    known_good=$(read_state)

    cd "$CHECKOUT"

    # Record current as known-good if not set
    if [[ -z "$known_good" ]]; then
        known_good=$(current_commit)
        write_state "$known_good"
        log "Initial known-good: $(short_hash "$known_good")"
    fi

    # Check candidate exists
    if ! git cat-file -e "$candidate^{commit}" 2>/dev/null; then
        log "ERROR: candidate '$candidate' is not a valid commit"
        return 1
    fi

    local candidate_hash
    candidate_hash=$(git rev-parse "$candidate")

    if [[ "$candidate_hash" == "$(current_commit)" ]]; then
        log "Already at candidate $(short_hash "$candidate_hash"), just restarting"
    else
        log "Deploying candidate: $(short_hash "$candidate_hash")"
        log "Known-good: $(short_hash "$known_good")"

        git checkout "$candidate_hash"
    fi

    log "Verifying channel dependencies..."
    if ! verify_deps; then
        log "ERROR: aborting deploy — dependencies missing; service untouched"
        git checkout "$known_good"
        log "Checkout rolled back to known-good $(short_hash "$known_good")"
        return 1
    fi

    log "Restarting service..."
    restart_service
    local restart_ts
    restart_ts=$(date '+%Y-%m-%d %H:%M:%S')

    log "Waiting ${STABILIZE_SEC}s for stabilization..."
    sleep "$STABILIZE_SEC"

    if ! service_active; then
        log "ERROR: service not active after restart"
        cmd_rollback_internal "$known_good" "$candidate_hash"
        return 1
    fi

    log "Running health check..."
    if health_check; then
        log "Health check PASSED"
    else
        log "ERROR: health check FAILED"
        cmd_rollback_internal "$known_good" "$candidate_hash"
        return 1
    fi

    log "Running channel check..."
    if channel_check_wait "$restart_ts"; then
        write_state "$candidate_hash"
        log "Promoted $(short_hash "$candidate_hash") to known-good"
    else
        log "ERROR: channel check FAILED"
        cmd_rollback_internal "$known_good" "$candidate_hash"
        return 1
    fi
}

cmd_rollback_internal() {
    local target="$1"
    local failed="${2:-unknown}"

    log "ROLLING BACK to $(short_hash "$target")"

    cd "$CHECKOUT"
    git checkout "$target"
    restart_service
    local restart_ts
    restart_ts=$(date '+%Y-%m-%d %H:%M:%S')

    sleep "$STABILIZE_SEC"

    if service_active && health_check && channel_check_wait "$restart_ts"; then
        log "Rollback successful, service healthy"
        # Write failure report
        cat > "/home/claw/.nanobot/deploy-failure.json" <<EOF
{
  "failed_candidate": "$failed",
  "known_good": "$target",
  "timestamp": "$(date -Iseconds)",
  "rollback_successful": true
}
EOF
    else
        log "CRITICAL: rollback also failed! Manual intervention required."
        cat > "/home/claw/.nanobot/deploy-failure.json" <<EOF
{
  "failed_candidate": "$failed",
  "known_good": "$target",
  "timestamp": "$(date -Iseconds)",
  "rollback_successful": false
}
EOF
        return 1
    fi
}

cmd_rollback() {
    local known_good
    known_good=$(read_state)
    if [[ -z "$known_good" ]]; then
        log "No known-good recorded, cannot rollback"
        return 1
    fi
    cmd_rollback_internal "$known_good" "$(current_commit)"
}

# ---- main ----

case "${1:-status}" in
    status)   cmd_status ;;
    rollback) cmd_rollback ;;
    *)        cmd_deploy "$1" ;;
esac