Files
archie/scripts/archie-update-watchdog
T

258 lines
9.2 KiB
Bash
Executable File

#!/usr/bin/env bash
# archie-update-watchdog -- restarts archied after an update, verifies health,
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
#
# Runtime-only updates still restart archied: the restart is the existing
# phase-2 handoff that makes the daemon consume and deliver the health report.
# It does not rebuild or reinstall the gateway binary.
set -uo pipefail
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
journal="$STATE_DIR/transaction"
mkdir -p "$STATE_DIR"
exec 9>"$STATE_DIR/lock"
if [ "${1:-}" = --recover ]; then
flock -n 9 || exit 0 # A live installer/watchdog owns recovery.
else
flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; }
fi
[ -d "$journal" ] || exit 0
printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder"
source "$journal/environment"
BIN_DIR="$ARCHIE_BIN_DIR"
CONFIG_PATH="$ARCHIE_CONFIG_PATH"
DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT"
AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE"
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
HEALTH_URL="${HEALTH_URL%/}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}"
POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}"
COMPONENTS="$ARCHIE_UPDATE_COMPONENTS"
UNITS="$ARCHIE_UPDATE_UNITS"
BINARIES="$ARCHIE_UPDATE_BINARIES"
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
phase() {
printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" &&
mv "$journal/phase.new" "$journal/phase" && sync -f "$journal"
}
json_escape() {
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
}
report_versions() {
case "$COMPONENTS" in
daemon)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
;;
agent)
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
daemon,agent)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
return 1
;;
esac
}
write_report() {
local health_check="$1" rolled_back="$2"
[ -z "$REPORT_PATH" ] && return 0
report_versions || return 1
mkdir -p "$(dirname "$REPORT_PATH")"
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
cat >"$tmp" <<EOF
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
EOF
chmod 600 "$tmp"
mv -f "$tmp" "$REPORT_PATH"
}
healthy() {
local report
report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 1
[ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] &&
[ "$(jq -r '.status' <<< "$report")" = ok ]
}
wait_healthy() {
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if healthy; then
deadline=$((SECONDS + OBSERVE_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
sleep "$POLL_INTERVAL_SECONDS"
healthy || return 1
done
return 0
fi
sleep "$POLL_INTERVAL_SECONDS"
done
return 1
}
manage_unit() { systemctl --user "$1" "$2.service"; }
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
# costs no process and no dependency, and accepting is exactly the signal this
# needs: the failure it guards against is `dial tcp ...: connect: connection
# refused`, which is the port not being bound at all.
port_accepts() {
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
}
# Wait for the dependency before starting its callers.
wait_for_state_store() {
[ -n "$STATE_STORE_TARGET" ] || {
echo "==> no State Store target known; starting the rest without waiting for it" >&2
return 0
}
local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if port_accepts "$STATE_STORE_TARGET"; then
echo "==> $STATE_STORE_TARGET accepts connections"
return 0
fi
sleep 1
done
echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2
return 0
}
# start_units brings the deployment up in dependency order: the State Store
# before anything that dials it, the Gateway before archied and the UI. The order
# sequences the calls; wait_for_state_store is what makes it mean "ready".
start_units() {
local unit rc=0
for unit in $UNITS; do
echo "==> restarting $unit.service"
manage_unit restart "$unit" || rc=1
if [ "$unit" = archie-state-store ]; then
wait_for_state_store
fi
done
return $rc
}
# stop_units tears down in reverse, so consumers are gone before the services
# they depend on.
stop_units() {
local unit ordered="" rc=0
for unit in $UNITS; do
ordered="$unit${ordered:+ $ordered}"
done
for unit in $ordered; do
manage_unit stop "$unit" || rc=1
done
return $rc
}
rollback_files() {
local cmd
for cmd in $BINARIES; do
if [ -f "$journal/previous/$cmd" ]; then
cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" &&
mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1
else
rm -f "$BIN_DIR/$cmd" || return 1
fi
rm -f "$BIN_DIR/.$cmd.new"
done
if [[ ",$COMPONENTS," == *,agent,* ]]; then
if [ -f "$journal/agent.version" ]; then
cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" &&
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
else
rm -f "$AGENT_VERSION_FILE" || return 1
fi
fi
sync -f "$BIN_DIR"
}
finish() {
phase "$1" || return 1
# A terminal journal is harmless if power fails before removal.
rm -rf "$STATE_DIR/finished"
mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1
rm -rf "$STATE_DIR/finished"
}
rollback() {
phase rolling-back || return 1
stop_units && rollback_files || return 1
if [ -f "$journal/image.json" ]; then
manage_unit start archie-state-store && wait_for_state_store &&
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1
fi
phase rolled-back || return 1
start_units || return 1
write_report failed true || return 1
finish rolled-back
}
promote() {
if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then
local status
status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2
fi
phase switching || return 1
stop_units || return 1
for cmd in $BINARIES; do
mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1
done
if [[ ",$COMPONENTS," == *,agent,* ]]; then
printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" &&
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
fi
sync -f "$BIN_DIR" || return 1
if [ -f "$journal/image.json" ]; then
manage_unit start archie-state-store && wait_for_state_store &&
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1
fi
phase observing && start_units && wait_healthy
}
# explain_database_recovery states what a daemon rollback does not do. The
# PostgreSQL database is never restored automatically: goose Down drops tables,
# and a restore is destructive, loses every write after the snapshot and needs
# every service stopped, so it is an operator's decision, not this script's.
explain_database_recovery() {
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
}
case "$(cat "$journal/phase")" in
committed) start_units && finish committed; exit $? ;;
rolled-back) start_units && finish rolled-back; exit $? ;;
esac
if [ -z "${1:-}" ] && promote; then
write_report passed false && finish committed
exit $?
fi
if [ "$(cat "$journal/phase")" = prepared ]; then
echo "update was not promoted; leaving running work alone" >&2
for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done
write_report failed false && finish rolled-back || exit 1
if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi
exit 1
fi
echo "==> restoring interrupted or unhealthy update" >&2
if rollback; then
if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi
exit 0
fi
echo "rollback incomplete; retained journal: $journal" >&2
exit 1