Files
archie/scripts/archie-update-watchdog
T
sam 99afccc3b2 fix(update): install and cycle every process in the release
A self-update to 1.23.0 failed its health check and rolled back. The
adapter built and installed ./cmd/archied and nothing else, and the
watchdog restarted archied.service and nothing else, so the new daemon
came up against a State Store, Gateway and UI still on the previous
release. Since 1.23.0 archied cannot boot without a reachable State
Store (openStateStoreAdapter), which on a host that had never run one is
an immediate exit, a failed health check, and a rollback that looks like
a bad release rather than an incomplete deployment.

Install every binary the reference deployment runs, versioned off the
archied tag, and back each up so the rollback restores all of them
together. A binary with no .prev was introduced by the failed release, so
rollback removes it instead of leaving one behind that points at
processes the restored release does not run.

Restart in dependency order: the State Store owns archie.db and every
other process dials it, the Gateway serves the contract archied and the
UI dial, so each comes up before its consumers and stops after them.
archied itself is cycled unconditionally and a failed stop still aborts
the rollback, because a presence probe that guesses wrong must never
skip the stop that keeps a live process off the binary and the database
being restored. The siblings are best-effort, which is safe because the
adapter now refuses the update outright, before touching the host, when
a unit for one of them is missing, naming what to create.

install.sh had the same single-binary assumption, so a fresh install
produced a host that could not start either.

Closes archie-core-hbqk.
2026-09-12 09:52:01 +10:00

211 lines
7.6 KiB
Bash
Executable File

#!/usr/bin/env bash
# archie-update-watchdog -- restarts archied after an update, verifies health,
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
#
# Runtime-only updates still restart archied: the restart is the existing
# phase-2 handoff that makes the daemon consume and deliver the health report.
# It does not rebuild or reinstall the gateway binary.
set -uo pipefail
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
# The daemon exports its own health address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# itself serves /healthz on -- [health] listen, defaulted in
# internal/infrastructure/configuration/defaults.go.
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
HEALTH_URL="${HEALTH_URL%/}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
POLL_INTERVAL_SECONDS=3
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
# Set by archie-update-install: the units to restart, in start order, and the
# binaries it replaced. The defaults keep a hand-driven run on a pre-
# decomposition host working.
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
json_escape() {
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
}
report_versions() {
case "$COMPONENTS" in
daemon)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
;;
agent)
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
daemon,agent)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
return 1
;;
esac
}
write_report() {
local health_check="$1" rolled_back="$2"
[ -z "$REPORT_PATH" ] && return 0
report_versions || return 1
mkdir -p "$(dirname "$REPORT_PATH")"
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
cat >"$tmp" <<EOF
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
EOF
chmod 600 "$tmp"
mv -f "$tmp" "$REPORT_PATH"
}
wait_healthy() {
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
return 0
fi
sleep "$POLL_INTERVAL_SECONDS"
done
return 1
}
# manage_unit runs one systemctl action. archied is required: this whole
# mechanism exists to restart it, and a failed stop must abort the rollback
# rather than let it race a live process over the binary and the database.
# The sibling processes are best-effort, because archie-update-install has
# already refused the update if a unit for one of them was missing, and a
# probe that guesses wrong must not silently skip the required stop.
manage_unit() {
local action="$1" unit="$2"
if [ "$unit" = archied ]; then
systemctl --user "$action" "$unit.service"
return
fi
systemctl --user "$action" "$unit.service" || true
}
# start_units brings the deployment up in dependency order: the State Store
# before anything that dials it, the Gateway before archied and the UI.
start_units() {
local unit rc=0
for unit in $UNITS; do
echo "==> restarting $unit.service"
manage_unit restart "$unit" || rc=1
done
return $rc
}
# stop_units tears down in reverse, so consumers are gone before the services
# they depend on.
stop_units() {
local unit ordered="" rc=0
for unit in $UNITS; do
ordered="$unit${ordered:+ $ordered}"
done
for unit in $ordered; do
manage_unit stop "$unit" || rc=1
done
return $rc
}
# rollback_gateway restores every binary the install replaced. A binary with
# no .prev was introduced by the failed release, so it is removed rather than
# left behind pointing at processes the restored release does not run.
rollback_gateway() {
local cmd rc=0
for cmd in $BINARIES; do
if [ -f "$BIN_DIR/$cmd.prev" ]; then
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
else
rm -f "$BIN_DIR/$cmd" || rc=1
fi
done
return $rc
}
rollback_agent_image() {
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
return 1
fi
docker image load --input "$AGENT_IMAGE_BACKUP"
}
rollback_task_db() {
if [ -z "$TASK_DB_PATH" ]; then
echo "==> no task database path; cannot restore schema-compatible state" >&2
return 1
fi
if [ ! -f "$TASK_DB_BACKUP" ]; then
echo "==> no task database backup; cannot restore schema-compatible state" >&2
return 1
fi
sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH"
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
start_units
if wait_healthy; then
echo "==> new version is healthy"
write_report "passed" false
exit 0
fi
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
rolled_back=false
# Stop before touching either the executable or its database. The service
# uses Restart=on-failure in the supported unit, so copying rollback state
# while a crash-loop child is alive can race with SQLite and execute the new
# binary after its files have been replaced.
if stop_units; then
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
echo "==> no backup for archied; cannot roll it back automatically" >&2
fi
;;
agent)
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
rollback_agent_image && rolled_back=true
else
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
fi
;;
daemon,agent)
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
fi
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
;;
esac
else
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
fi
if [ "$rolled_back" = true ]; then
start_units
fi
write_report "failed" "$rolled_back"