#!/usr/bin/env bash # archie-update-watchdog -- restarts archied after an update, verifies health, # and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS. # # Runtime-only updates still restart archied: the restart is the existing # phase-2 handoff that makes the daemon consume and deliver the health report. # It does not rebuild or reinstall the gateway binary. set -uo pipefail BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}" AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" # The daemon exports its own health address (releaseupdate.CommandInstaller); # this default only covers a hand-driven run, and must stay the address archied # itself serves /healthz on -- [health] listen, defaulted in # internal/infrastructure/configuration/defaults.go. HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}" HEALTH_URL="${HEALTH_URL%/}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" POLL_INTERVAL_SECONDS=3 COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}" # Set by archie-update-install: the units to restart, in start order, and the # binaries it replaced. The defaults keep a hand-driven run on a pre- # decomposition host working. UNITS="${ARCHIE_UPDATE_UNITS:-archied}" BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}" json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' } report_versions() { case "$COMPONENTS" in daemon) previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}" installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}" ;; agent) previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" ;; daemon,agent) previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" ;; *) echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 return 1 ;; esac } write_report() { local health_check="$1" rolled_back="$2" [ -z "$REPORT_PATH" ] && return 0 report_versions || return 1 mkdir -p "$(dirname "$REPORT_PATH")" tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")" cat >"$tmp" < restarting $unit.service" manage_unit restart "$unit" || rc=1 done return $rc } # stop_units tears down in reverse, so consumers are gone before the services # they depend on. stop_units() { local unit ordered="" rc=0 for unit in $UNITS; do ordered="$unit${ordered:+ $ordered}" done for unit in $ordered; do manage_unit stop "$unit" || rc=1 done return $rc } # rollback_gateway restores every binary the install replaced. A binary with # no .prev was introduced by the failed release, so it is removed rather than # left behind pointing at processes the restored release does not run. rollback_gateway() { local cmd rc=0 for cmd in $BINARIES; do if [ -f "$BIN_DIR/$cmd.prev" ]; then cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1 else rm -f "$BIN_DIR/$cmd" || rc=1 fi done return $rc } rollback_agent_image() { if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 return 1 fi docker image load --input "$AGENT_IMAGE_BACKUP" } # The sidecar describes the image restored above. Restore its prior state in # the same rollback path; a first install has no prior file and must remove the # newly-created metadata rather than leave a false installed version behind. rollback_agent_version() { if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then [ -f "$AGENT_VERSION_BACKUP" ] || return 1 cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE" else rm -f "$AGENT_VERSION_FILE" fi } rollback_task_db() { if [ -z "$TASK_DB_PATH" ]; then echo "==> no task database path; cannot restore schema-compatible state" >&2 return 1 fi if [ ! -f "$TASK_DB_BACKUP" ]; then echo "==> no task database backup; cannot restore schema-compatible state" >&2 return 1 fi sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH" rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm" } start_units if wait_healthy; then echo "==> new version is healthy" write_report "passed" false exit 0 fi echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components" rolled_back=false # Stop before touching either the executable or its database. The service # uses Restart=on-failure in the supported unit, so copying rollback state # while a crash-loop child is alive can race with SQLite and execute the new # binary after its files have been replaced. if stop_units; then case "$COMPONENTS" in daemon) if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then rolled_back=true fi else echo "==> no backup for archied; cannot roll it back automatically" >&2 fi ;; agent) if [ -f "$AGENT_IMAGE_BACKUP" ]; then if rollback_agent_image && rollback_agent_version; then rolled_back=true fi else echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 fi ;; daemon,agent) # Preserve the old all-or-nothing rollback guarantee: never restore one # component and leave the other on the failed release. if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then if rollback_agent_image && rollback_agent_version && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then rolled_back=true fi else echo "==> not all changed-component backups exist; cannot roll back automatically" >&2 fi ;; *) echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 ;; esac else echo "==> could not stop every unit; refusing rollback while one may be running" >&2 fi if [ "$rolled_back" = true ]; then start_units fi write_report "failed" "$rolled_back"