The pre-update PostgreSQL snapshot is the only backup; the watchdog restores binaries and never the database.
276 lines
11 KiB
Bash
Executable File
276 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# archie-update-watchdog -- restarts archied after an update, verifies health,
|
|
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
|
|
#
|
|
# Runtime-only updates still restart archied: the restart is the existing
|
|
# phase-2 handoff that makes the daemon consume and deliver the health report.
|
|
# It does not rebuild or reinstall the gateway binary.
|
|
set -uo pipefail
|
|
|
|
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
|
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
|
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
|
|
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
|
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
|
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
|
|
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
|
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
|
# The daemon exports its own health address (releaseupdate.CommandInstaller);
|
|
# this default only covers a hand-driven run, and must stay the address archied
|
|
# itself serves /healthz on -- [health] listen, defaulted in
|
|
# internal/infrastructure/configuration/defaults.go.
|
|
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
|
|
HEALTH_URL="${HEALTH_URL%/}"
|
|
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
|
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
|
POLL_INTERVAL_SECONDS=3
|
|
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
|
|
# Set by archie-update-install: the units to restart, in start order, and the
|
|
# binaries it replaced. The defaults keep a hand-driven run on a pre-
|
|
# decomposition host working.
|
|
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
|
|
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
|
|
|
|
# The State Store's address, so the start order below can mean "ready" rather than
|
|
# "launched". Empty when the config names none or the installed updater predates
|
|
# passing one -- waiting is then skipped, which is the previous behaviour.
|
|
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
|
|
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
|
|
|
|
json_escape() {
|
|
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
report_versions() {
|
|
case "$COMPONENTS" in
|
|
daemon)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
|
|
;;
|
|
agent)
|
|
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
daemon,agent)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
*)
|
|
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
write_report() {
|
|
local health_check="$1" rolled_back="$2"
|
|
[ -z "$REPORT_PATH" ] && return 0
|
|
report_versions || return 1
|
|
mkdir -p "$(dirname "$REPORT_PATH")"
|
|
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
|
|
cat >"$tmp" <<EOF
|
|
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
|
|
EOF
|
|
chmod 600 "$tmp"
|
|
mv -f "$tmp" "$REPORT_PATH"
|
|
}
|
|
|
|
wait_healthy() {
|
|
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL_SECONDS"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# manage_unit runs one systemctl action. archied is required: this whole
|
|
# mechanism exists to restart it, and a failed stop must abort the rollback
|
|
# rather than let it race a live process over the binary and the database.
|
|
# The sibling processes are best-effort, because archie-update-install has
|
|
# already refused the update if a unit for one of them was missing, and a
|
|
# probe that guesses wrong must not silently skip the required stop.
|
|
manage_unit() {
|
|
local action="$1" unit="$2"
|
|
if [ "$unit" = archied ]; then
|
|
systemctl --user "$action" "$unit.service"
|
|
return
|
|
fi
|
|
systemctl --user "$action" "$unit.service" || true
|
|
}
|
|
|
|
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
|
|
# costs no process and no dependency, and accepting is exactly the signal this
|
|
# needs: the failure it guards against is `dial tcp ...: connect: connection
|
|
# refused`, which is the port not being bound at all.
|
|
port_accepts() {
|
|
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
|
|
}
|
|
|
|
# wait_for_state_store blocks until the State Store accepts connections, or the
|
|
# budget runs out.
|
|
#
|
|
# Ordering the units is not enough on its own: `systemctl --user start` returns
|
|
# once the process is forked, not once its port accepts. Measured on a 1.37.1 ->
|
|
# 1.38.0 update, the State Store's unit reported started while it was still
|
|
# binding, archied was started next, dialled a closed port and exited 1, and only
|
|
# systemd's Restart=on-failure brought it back seven seconds later. A host whose
|
|
# unit lacks that restart policy gets a failed update behind a healthy-looking
|
|
# "build and install succeeded" line.
|
|
#
|
|
# A timeout does NOT fail the update. If the State Store never comes up, archied
|
|
# fails closed on its own and wait_healthy rolls the release back -- waiting is an
|
|
# optimisation for the ordinary case, and losing the race must stay a slower path
|
|
# to the same place rather than a second failure mode.
|
|
wait_for_state_store() {
|
|
[ -n "$STATE_STORE_TARGET" ] || {
|
|
echo "==> no State Store target known; starting the rest without waiting for it" >&2
|
|
return 0
|
|
}
|
|
local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if port_accepts "$STATE_STORE_TARGET"; then
|
|
echo "==> $STATE_STORE_TARGET accepts connections"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2
|
|
return 0
|
|
}
|
|
|
|
# start_units brings the deployment up in dependency order: the State Store
|
|
# before anything that dials it, the Gateway before archied and the UI. The order
|
|
# sequences the calls; wait_for_state_store is what makes it mean "ready".
|
|
start_units() {
|
|
local unit rc=0
|
|
for unit in $UNITS; do
|
|
echo "==> restarting $unit.service"
|
|
manage_unit restart "$unit" || rc=1
|
|
if [ "$unit" = archie-state-store ]; then
|
|
wait_for_state_store
|
|
fi
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# stop_units tears down in reverse, so consumers are gone before the services
|
|
# they depend on.
|
|
stop_units() {
|
|
local unit ordered="" rc=0
|
|
for unit in $UNITS; do
|
|
ordered="$unit${ordered:+ $ordered}"
|
|
done
|
|
for unit in $ordered; do
|
|
manage_unit stop "$unit" || rc=1
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# rollback_gateway restores every binary the install replaced. A binary with
|
|
# no .prev was introduced by the failed release, so it is removed rather than
|
|
# left behind pointing at processes the restored release does not run.
|
|
rollback_gateway() {
|
|
local cmd rc=0
|
|
for cmd in $BINARIES; do
|
|
if [ -f "$BIN_DIR/$cmd.prev" ]; then
|
|
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
|
|
else
|
|
rm -f "$BIN_DIR/$cmd" || rc=1
|
|
fi
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
rollback_agent_image() {
|
|
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
|
|
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
|
return 1
|
|
fi
|
|
docker image load --input "$AGENT_IMAGE_BACKUP"
|
|
}
|
|
|
|
# The sidecar describes the image restored above. Restore its prior state in
|
|
# the same rollback path; a first install has no prior file and must remove the
|
|
# newly-created metadata rather than leave a false installed version behind.
|
|
rollback_agent_version() {
|
|
if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then
|
|
[ -f "$AGENT_VERSION_BACKUP" ] || return 1
|
|
cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE"
|
|
else
|
|
rm -f "$AGENT_VERSION_FILE"
|
|
fi
|
|
}
|
|
|
|
# explain_database_recovery states what a daemon rollback does not do. The
|
|
# PostgreSQL database is never restored automatically: goose Down drops tables,
|
|
# and a restore is destructive, loses every write after the snapshot and needs
|
|
# every service stopped, so it is an operator's decision, not this script's.
|
|
explain_database_recovery() {
|
|
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
|
|
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
|
|
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
|
}
|
|
|
|
start_units
|
|
|
|
if wait_healthy; then
|
|
echo "==> new version is healthy"
|
|
write_report "passed" false
|
|
exit 0
|
|
fi
|
|
|
|
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
|
|
rolled_back=false
|
|
# Stop before touching the executable. The service uses Restart=on-failure in
|
|
# the supported unit, so copying rollback state while a crash-loop child is
|
|
# alive can execute the new binary after its files have been replaced.
|
|
if stop_units; then
|
|
case "$COMPONENTS" in
|
|
daemon)
|
|
if [ -f "$BIN_DIR/archied.prev" ]; then
|
|
if rollback_gateway; then
|
|
rolled_back=true
|
|
fi
|
|
else
|
|
echo "==> no backup for archied; cannot roll it back automatically" >&2
|
|
fi
|
|
;;
|
|
agent)
|
|
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
|
if rollback_agent_image && rollback_agent_version; then
|
|
rolled_back=true
|
|
fi
|
|
else
|
|
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
|
fi
|
|
;;
|
|
daemon,agent)
|
|
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
|
# component and leave the other on the failed release.
|
|
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
|
if rollback_agent_image && rollback_agent_version && rollback_gateway; then
|
|
rolled_back=true
|
|
fi
|
|
else
|
|
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
|
|
fi
|
|
;;
|
|
*)
|
|
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
|
;;
|
|
esac
|
|
else
|
|
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
|
|
fi
|
|
if [ "$rolled_back" = true ]; then
|
|
start_units
|
|
fi
|
|
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
|
|
explain_database_recovery
|
|
fi
|
|
|
|
write_report "failed" "$rolled_back"
|