258 lines
9.2 KiB
Bash
Executable File
258 lines
9.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# archie-update-watchdog -- restarts archied after an update, verifies health,
|
|
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
|
|
#
|
|
# Runtime-only updates still restart archied: the restart is the existing
|
|
# phase-2 handoff that makes the daemon consume and deliver the health report.
|
|
# It does not rebuild or reinstall the gateway binary.
|
|
set -uo pipefail
|
|
|
|
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
|
|
journal="$STATE_DIR/transaction"
|
|
mkdir -p "$STATE_DIR"
|
|
exec 9>"$STATE_DIR/lock"
|
|
if [ "${1:-}" = --recover ]; then
|
|
flock -n 9 || exit 0 # A live installer/watchdog owns recovery.
|
|
else
|
|
flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; }
|
|
fi
|
|
[ -d "$journal" ] || exit 0
|
|
printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder"
|
|
source "$journal/environment"
|
|
BIN_DIR="$ARCHIE_BIN_DIR"
|
|
CONFIG_PATH="$ARCHIE_CONFIG_PATH"
|
|
DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT"
|
|
AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE"
|
|
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
|
|
HEALTH_URL="${HEALTH_URL%/}"
|
|
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
|
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
|
OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}"
|
|
POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}"
|
|
COMPONENTS="$ARCHIE_UPDATE_COMPONENTS"
|
|
UNITS="$ARCHIE_UPDATE_UNITS"
|
|
BINARIES="$ARCHIE_UPDATE_BINARIES"
|
|
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
|
|
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
|
|
|
|
phase() {
|
|
printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" &&
|
|
mv "$journal/phase.new" "$journal/phase" && sync -f "$journal"
|
|
}
|
|
|
|
json_escape() {
|
|
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
report_versions() {
|
|
case "$COMPONENTS" in
|
|
daemon)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
|
|
;;
|
|
agent)
|
|
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
daemon,agent)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
*)
|
|
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
write_report() {
|
|
local health_check="$1" rolled_back="$2"
|
|
[ -z "$REPORT_PATH" ] && return 0
|
|
report_versions || return 1
|
|
mkdir -p "$(dirname "$REPORT_PATH")"
|
|
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
|
|
cat >"$tmp" <<EOF
|
|
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
|
|
EOF
|
|
chmod 600 "$tmp"
|
|
mv -f "$tmp" "$REPORT_PATH"
|
|
}
|
|
|
|
healthy() {
|
|
local report
|
|
report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 1
|
|
[ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] &&
|
|
[ "$(jq -r '.status' <<< "$report")" = ok ]
|
|
}
|
|
|
|
wait_healthy() {
|
|
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if healthy; then
|
|
deadline=$((SECONDS + OBSERVE_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
sleep "$POLL_INTERVAL_SECONDS"
|
|
healthy || return 1
|
|
done
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL_SECONDS"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
manage_unit() { systemctl --user "$1" "$2.service"; }
|
|
|
|
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
|
|
# costs no process and no dependency, and accepting is exactly the signal this
|
|
# needs: the failure it guards against is `dial tcp ...: connect: connection
|
|
# refused`, which is the port not being bound at all.
|
|
port_accepts() {
|
|
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
|
|
}
|
|
|
|
# Wait for the dependency before starting its callers.
|
|
wait_for_state_store() {
|
|
[ -n "$STATE_STORE_TARGET" ] || {
|
|
echo "==> no State Store target known; starting the rest without waiting for it" >&2
|
|
return 0
|
|
}
|
|
local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if port_accepts "$STATE_STORE_TARGET"; then
|
|
echo "==> $STATE_STORE_TARGET accepts connections"
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2
|
|
return 0
|
|
}
|
|
|
|
# start_units brings the deployment up in dependency order: the State Store
|
|
# before anything that dials it, the Gateway before archied and the UI. The order
|
|
# sequences the calls; wait_for_state_store is what makes it mean "ready".
|
|
start_units() {
|
|
local unit rc=0
|
|
for unit in $UNITS; do
|
|
echo "==> restarting $unit.service"
|
|
manage_unit restart "$unit" || rc=1
|
|
if [ "$unit" = archie-state-store ]; then
|
|
wait_for_state_store
|
|
fi
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# stop_units tears down in reverse, so consumers are gone before the services
|
|
# they depend on.
|
|
stop_units() {
|
|
local unit ordered="" rc=0
|
|
for unit in $UNITS; do
|
|
ordered="$unit${ordered:+ $ordered}"
|
|
done
|
|
for unit in $ordered; do
|
|
manage_unit stop "$unit" || rc=1
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
rollback_files() {
|
|
local cmd
|
|
for cmd in $BINARIES; do
|
|
if [ -f "$journal/previous/$cmd" ]; then
|
|
cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" &&
|
|
mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1
|
|
else
|
|
rm -f "$BIN_DIR/$cmd" || return 1
|
|
fi
|
|
rm -f "$BIN_DIR/.$cmd.new"
|
|
done
|
|
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
|
if [ -f "$journal/agent.version" ]; then
|
|
cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" &&
|
|
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
|
else
|
|
rm -f "$AGENT_VERSION_FILE" || return 1
|
|
fi
|
|
fi
|
|
sync -f "$BIN_DIR"
|
|
}
|
|
|
|
finish() {
|
|
phase "$1" || return 1
|
|
# A terminal journal is harmless if power fails before removal.
|
|
rm -rf "$STATE_DIR/finished"
|
|
mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1
|
|
rm -rf "$STATE_DIR/finished"
|
|
}
|
|
|
|
rollback() {
|
|
phase rolling-back || return 1
|
|
stop_units && rollback_files || return 1
|
|
if [ -f "$journal/image.json" ]; then
|
|
manage_unit start archie-state-store && wait_for_state_store &&
|
|
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1
|
|
fi
|
|
phase rolled-back || return 1
|
|
start_units || return 1
|
|
write_report failed true || return 1
|
|
finish rolled-back
|
|
}
|
|
|
|
promote() {
|
|
if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then
|
|
local status
|
|
status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2
|
|
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2
|
|
fi
|
|
phase switching || return 1
|
|
stop_units || return 1
|
|
for cmd in $BINARIES; do
|
|
mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1
|
|
done
|
|
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
|
printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" &&
|
|
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
|
fi
|
|
sync -f "$BIN_DIR" || return 1
|
|
if [ -f "$journal/image.json" ]; then
|
|
manage_unit start archie-state-store && wait_for_state_store &&
|
|
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1
|
|
fi
|
|
phase observing && start_units && wait_healthy
|
|
}
|
|
|
|
# explain_database_recovery states what a daemon rollback does not do. The
|
|
# PostgreSQL database is never restored automatically: goose Down drops tables,
|
|
# and a restore is destructive, loses every write after the snapshot and needs
|
|
# every service stopped, so it is an operator's decision, not this script's.
|
|
explain_database_recovery() {
|
|
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
|
|
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
|
|
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
|
}
|
|
|
|
case "$(cat "$journal/phase")" in
|
|
committed) start_units && finish committed; exit $? ;;
|
|
rolled-back) start_units && finish rolled-back; exit $? ;;
|
|
esac
|
|
if [ -z "${1:-}" ] && promote; then
|
|
write_report passed false && finish committed
|
|
exit $?
|
|
fi
|
|
if [ "$(cat "$journal/phase")" = prepared ]; then
|
|
echo "update was not promoted; leaving running work alone" >&2
|
|
for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done
|
|
write_report failed false && finish rolled-back || exit 1
|
|
if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi
|
|
exit 1
|
|
fi
|
|
echo "==> restoring interrupted or unhealthy update" >&2
|
|
if rollback; then
|
|
if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi
|
|
exit 0
|
|
fi
|
|
echo "rollback incomplete; retained journal: $journal" >&2
|
|
exit 1
|