Files
archie/scripts/archie-update-watchdog
T
sam aac31b69ad docs(agents): a bead never carries production instance detail
Beads sync to a Dolt remote, so anything written into one is published.
Two beads filed earlier today named an operator host, its internal
address and its filesystem paths -- the tracker was treated as a private
store, which it is not.

The rule is stated where the tracker is described, next to the existing
one about .beads/ never reaching a commit, because it is the same class:
the tracker is not a private store. Host-specific facts belong in the
operator's memos instance, which a bead may point at.
2026-09-23 12:29:29 +10:00

278 lines
11 KiB
Bash
Executable File

#!/usr/bin/env bash
# archie-update-watchdog -- restarts archied after an update, verifies health,
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
#
# Runtime-only updates still restart archied: the restart is the existing
# phase-2 handoff that makes the daemon consume and deliver the health report.
# It does not rebuild or reinstall the gateway binary.
set -uo pipefail
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
# The daemon exports its own health address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# itself serves /healthz on -- [health] listen, defaulted in
# internal/infrastructure/configuration/defaults.go.
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
HEALTH_URL="${HEALTH_URL%/}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
POLL_INTERVAL_SECONDS=3
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
# Set by archie-update-install: the units to restart, in start order, and the
# binaries it replaced. The defaults keep a hand-driven run on a pre-
# decomposition host working.
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
# The State Store's address, so the start order below can mean "ready" rather than
# "launched". Empty when the config names none or the installed updater predates
# passing one -- waiting is then skipped, which is the previous behaviour.
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
json_escape() {
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
}
report_versions() {
case "$COMPONENTS" in
daemon)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
;;
agent)
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
daemon,agent)
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
return 1
;;
esac
}
write_report() {
local health_check="$1" rolled_back="$2"
[ -z "$REPORT_PATH" ] && return 0
report_versions || return 1
mkdir -p "$(dirname "$REPORT_PATH")"
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
cat >"$tmp" <<EOF
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
EOF
chmod 600 "$tmp"
mv -f "$tmp" "$REPORT_PATH"
}
wait_healthy() {
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
return 0
fi
sleep "$POLL_INTERVAL_SECONDS"
done
return 1
}
# manage_unit runs one systemctl action. archied is required: this whole
# mechanism exists to restart it, and a failed stop must abort the rollback
# rather than let it race a live process over the binary and the database.
# The sibling processes are best-effort, because archie-update-install has
# already refused the update if a unit for one of them was missing, and a
# probe that guesses wrong must not silently skip the required stop.
manage_unit() {
local action="$1" unit="$2"
if [ "$unit" = archied ]; then
systemctl --user "$action" "$unit.service"
return
fi
systemctl --user "$action" "$unit.service" || true
}
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
# costs no process and no dependency, and accepting is exactly the signal this
# needs: the failure it guards against is `dial tcp ...: connect: connection
# refused`, which is the port not being bound at all.
port_accepts() {
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
}
# wait_for_state_store blocks until the State Store accepts connections, or the
# budget runs out.
#
# Ordering the units is not enough on its own: `systemctl --user start` returns
# once the process is forked, not once its port accepts. Measured on a 1.37.1 ->
# 1.38.0 update, the State Store's unit reported started while it was still
# binding, archied was started next, dialled a closed port and exited 1, and only
# systemd's Restart=on-failure brought it back seven seconds later. A host whose
# unit lacks that restart policy gets a failed update behind a healthy-looking
# "build and install succeeded" line.
#
# A timeout does NOT fail the update. If the State Store never comes up, archied
# fails closed on its own and wait_healthy rolls the release back -- waiting is an
# optimisation for the ordinary case, and losing the race must stay a slower path
# to the same place rather than a second failure mode.
wait_for_state_store() {
[ -n "$STATE_STORE_TARGET" ] || {
echo "==> no State Store target known; starting the rest without waiting for it" >&2
return 0
}
local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS))
while [ "$SECONDS" -lt "$deadline" ]; do
if port_accepts "$STATE_STORE_TARGET"; then
echo "==> $STATE_STORE_TARGET accepts connections"
return 0
fi
sleep 1
done
echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2
return 0
}
# start_units brings the deployment up in dependency order: the State Store
# before anything that dials it, the Gateway before archied and the UI. The order
# sequences the calls; wait_for_state_store is what makes it mean "ready".
start_units() {
local unit rc=0
for unit in $UNITS; do
echo "==> restarting $unit.service"
manage_unit restart "$unit" || rc=1
if [ "$unit" = archie-state-store ]; then
wait_for_state_store
fi
done
return $rc
}
# stop_units tears down in reverse, so consumers are gone before the services
# they depend on.
stop_units() {
local unit ordered="" rc=0
for unit in $UNITS; do
ordered="$unit${ordered:+ $ordered}"
done
for unit in $ordered; do
manage_unit stop "$unit" || rc=1
done
return $rc
}
# rollback_gateway restores every binary the install replaced. A binary with
# no .prev was introduced by the failed release, so it is removed rather than
# left behind pointing at processes the restored release does not run.
rollback_gateway() {
local cmd rc=0
for cmd in $BINARIES; do
if [ -f "$BIN_DIR/$cmd.prev" ]; then
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
else
rm -f "$BIN_DIR/$cmd" || rc=1
fi
done
return $rc
}
rollback_agent_image() {
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
return 1
fi
docker image load --input "$AGENT_IMAGE_BACKUP"
}
# The sidecar describes the image restored above. Restore its prior state in
# the same rollback path; a first install has no prior file and must remove the
# newly-created metadata rather than leave a false installed version behind.
rollback_agent_version() {
if [ "$AGENT_VERSION_PREVIOUS_EXISTS" = 1 ]; then
[ -f "$AGENT_VERSION_BACKUP" ] || return 1
cp -f "$AGENT_VERSION_BACKUP" "$AGENT_VERSION_FILE"
else
rm -f "$AGENT_VERSION_FILE"
fi
}
rollback_task_db() {
if [ -z "$TASK_DB_PATH" ]; then
echo "==> no task database path; cannot restore schema-compatible state" >&2
return 1
fi
if [ ! -f "$TASK_DB_BACKUP" ]; then
echo "==> no task database backup; cannot restore schema-compatible state" >&2
return 1
fi
sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH"
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
start_units
if wait_healthy; then
echo "==> new version is healthy"
write_report "passed" false
exit 0
fi
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
rolled_back=false
# Stop before touching either the executable or its database. The service
# uses Restart=on-failure in the supported unit, so copying rollback state
# while a crash-loop child is alive can race with SQLite and execute the new
# binary after its files have been replaced.
if stop_units; then
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
echo "==> no backup for archied; cannot roll it back automatically" >&2
fi
;;
agent)
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
if rollback_agent_image && rollback_agent_version; then
rolled_back=true
fi
else
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
fi
;;
daemon,agent)
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if rollback_agent_image && rollback_agent_version && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
fi
;;
*)
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
;;
esac
else
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
fi
if [ "$rolled_back" = true ]; then
start_units
fi
write_report "failed" "$rolled_back"