#!/usr/bin/env bash # archie-update-watchdog -- restarts archied after an update, verifies health, # and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS. # # Runtime-only updates still restart archied: the restart is the existing # phase-2 handoff that makes the daemon consume and deliver the health report. # It does not rebuild or reinstall the gateway binary. set -uo pipefail BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}" AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}" DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}" # The daemon exports its own health address (releaseupdate.CommandInstaller); # this default only covers a hand-driven run, and must stay the address archied # itself serves /healthz on -- [health] listen, defaulted in # internal/infrastructure/configuration/defaults.go. HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}" HEALTH_URL="${HEALTH_URL%/}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" POLL_INTERVAL_SECONDS=3 COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}" # Set by archie-update-install: the units to restart, in start order, and the # binaries it replaced. The defaults keep a hand-driven run on a pre- # decomposition host working. UNITS="${ARCHIE_UPDATE_UNITS:-archied}" BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}" # The State Store's address, so the start order below can mean "ready" rather than # "launched". Empty when the config names none or the installed updater predates # passing one -- waiting is then skipped, which is the previous behaviour. STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}" STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}" json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' } report_versions() { case "$COMPONENTS" in daemon) previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}" installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}" ;; agent) previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" ;; daemon,agent) previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" ;; *) echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 return 1 ;; esac } write_report() { local health_check="$1" rolled_back="$2" [ -z "$REPORT_PATH" ] && return 0 report_versions || return 1 mkdir -p "$(dirname "$REPORT_PATH")" tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")" cat >"$tmp" <"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null } # wait_for_state_store blocks until the State Store accepts connections, or the # budget runs out. # # Ordering the units is not enough on its own: `systemctl --user start` returns # once the process is forked, not once its port accepts. Measured on a 1.37.1 -> # 1.38.0 update, the State Store's unit reported started while it was still # binding, archied was started next, dialled a closed port and exited 1, and only # systemd's Restart=on-failure brought it back seven seconds later. A host whose # unit lacks that restart policy gets a failed update behind a healthy-looking # "build and install succeeded" line. # # A timeout does NOT fail the update. If the State Store never comes up, archied # fails closed on its own and wait_healthy rolls the release back -- waiting is an # optimisation for the ordinary case, and losing the race must stay a slower path # to the same place rather than a second failure mode. wait_for_state_store() { [ -n "$STATE_STORE_TARGET" ] || { echo "==> no State Store target known; starting the rest without waiting for it" >&2 return 0 } local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS)) while [ "$SECONDS" -lt "$deadline" ]; do if port_accepts "$STATE_STORE_TARGET"; then echo "==> $STATE_STORE_TARGET accepts connections" return 0 fi sleep 1 done echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2 return 0 } # start_units brings the deployment up in dependency order: the State Store # before anything that dials it, the Gateway before archied and the UI. The order # sequences the calls; wait_for_state_store is what makes it mean "ready". start_units() { local unit rc=0 for unit in $UNITS; do echo "==> restarting $unit.service" manage_unit restart "$unit" || rc=1 if [ "$unit" = archie-state-store ]; then wait_for_state_store fi done return $rc } # stop_units tears down in reverse, so consumers are gone before the services # they depend on. stop_units() { local unit ordered="" rc=0 for unit in $UNITS; do ordered="$unit${ordered:+ $ordered}" done for unit in $ordered; do manage_unit stop "$unit" || rc=1 done return $rc } # rollback_gateway restores every binary the install replaced. A binary with # no .prev was introduced by the failed release, so it is removed rather than # left behind pointing at processes the restored release does not run. rollback_gateway() { local cmd rc=0 for cmd in $BINARIES; do if [ -f "$BIN_DIR/$cmd.prev" ]; then cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1 else rm -f "$BIN_DIR/$cmd" || rc=1 fi done return $rc } rollback_agent_image() { if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 return 1 fi docker image load --input "$AGENT_IMAGE_BACKUP" } # The sidecar describes the image restored above. Restore its prior state in # the same rollback path; a first install has no prior file and must remove the # newly-created metadata rather than leave a false installed version behind. rollback_agent_version() { if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then [ -f "$AGENT_VERSION_BACKUP" ] || return 1 cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE" else rm -f "$AGENT_VERSION_FILE" fi } # explain_database_recovery states what a daemon rollback does not do. The # PostgreSQL database is never restored automatically: goose Down drops tables, # and a restore is destructive, loses every write after the snapshot and needs # every service stopped, so it is an operator's decision, not this script's. explain_database_recovery() { echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2 echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2 echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2 } start_units if wait_healthy; then echo "==> new version is healthy" write_report "passed" false exit 0 fi echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components" rolled_back=false # Stop before touching the executable. The service uses Restart=on-failure in # the supported unit, so copying rollback state while a crash-loop child is # alive can execute the new binary after its files have been replaced. if stop_units; then case "$COMPONENTS" in daemon) if [ -f "$BIN_DIR/archied.prev" ]; then if rollback_gateway; then rolled_back=true fi else echo "==> no backup for archied; cannot roll it back automatically" >&2 fi ;; agent) if [ -f "$AGENT_IMAGE_BACKUP" ]; then if rollback_agent_image && rollback_agent_version; then rolled_back=true fi else echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2 fi ;; daemon,agent) # Preserve the old all-or-nothing rollback guarantee: never restore one # component and leave the other on the failed release. if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then if rollback_agent_image && rollback_agent_version && rollback_gateway; then rolled_back=true fi else echo "==> not all changed-component backups exist; cannot roll back automatically" >&2 fi ;; *) echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 ;; esac else echo "==> could not stop every unit; refusing rollback while one may be running" >&2 fi if [ "$rolled_back" = true ]; then start_units fi if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery fi write_report "failed" "$rolled_back"