A self-update to 1.23.0 failed its health check and rolled back. The adapter built and installed ./cmd/archied and nothing else, and the watchdog restarted archied.service and nothing else, so the new daemon came up against a State Store, Gateway and UI still on the previous release. Since 1.23.0 archied cannot boot without a reachable State Store (openStateStoreAdapter), which on a host that had never run one is an immediate exit, a failed health check, and a rollback that looks like a bad release rather than an incomplete deployment. Install every binary the reference deployment runs, versioned off the archied tag, and back each up so the rollback restores all of them together. A binary with no .prev was introduced by the failed release, so rollback removes it instead of leaving one behind that points at processes the restored release does not run. Restart in dependency order: the State Store owns archie.db and every other process dials it, the Gateway serves the contract archied and the UI dial, so each comes up before its consumers and stops after them. archied itself is cycled unconditionally and a failed stop still aborts the rollback, because a presence probe that guesses wrong must never skip the stop that keeps a live process off the binary and the database being restored. The siblings are best-effort, which is safe because the adapter now refuses the update outright, before touching the host, when a unit for one of them is missing, naming what to create. install.sh had the same single-binary assumption, so a fresh install produced a host that could not start either. Closes archie-core-hbqk.
211 lines
7.6 KiB
Bash
Executable File
211 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# archie-update-watchdog -- restarts archied after an update, verifies health,
|
|
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
|
|
#
|
|
# Runtime-only updates still restart archied: the restart is the existing
|
|
# phase-2 handoff that makes the daemon consume and deliver the health report.
|
|
# It does not rebuild or reinstall the gateway binary.
|
|
set -uo pipefail
|
|
|
|
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
|
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
|
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
|
|
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
|
|
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
|
# The daemon exports its own health address (releaseupdate.CommandInstaller);
|
|
# this default only covers a hand-driven run, and must stay the address archied
|
|
# itself serves /healthz on -- [health] listen, defaulted in
|
|
# internal/infrastructure/configuration/defaults.go.
|
|
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
|
|
HEALTH_URL="${HEALTH_URL%/}"
|
|
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
|
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
|
POLL_INTERVAL_SECONDS=3
|
|
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
|
|
# Set by archie-update-install: the units to restart, in start order, and the
|
|
# binaries it replaced. The defaults keep a hand-driven run on a pre-
|
|
# decomposition host working.
|
|
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
|
|
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
|
|
|
|
json_escape() {
|
|
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
report_versions() {
|
|
case "$COMPONENTS" in
|
|
daemon)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
|
|
;;
|
|
agent)
|
|
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
daemon,agent)
|
|
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
|
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
|
;;
|
|
*)
|
|
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
write_report() {
|
|
local health_check="$1" rolled_back="$2"
|
|
[ -z "$REPORT_PATH" ] && return 0
|
|
report_versions || return 1
|
|
mkdir -p "$(dirname "$REPORT_PATH")"
|
|
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
|
|
cat >"$tmp" <<EOF
|
|
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
|
|
EOF
|
|
chmod 600 "$tmp"
|
|
mv -f "$tmp" "$REPORT_PATH"
|
|
}
|
|
|
|
wait_healthy() {
|
|
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL_SECONDS"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# manage_unit runs one systemctl action. archied is required: this whole
|
|
# mechanism exists to restart it, and a failed stop must abort the rollback
|
|
# rather than let it race a live process over the binary and the database.
|
|
# The sibling processes are best-effort, because archie-update-install has
|
|
# already refused the update if a unit for one of them was missing, and a
|
|
# probe that guesses wrong must not silently skip the required stop.
|
|
manage_unit() {
|
|
local action="$1" unit="$2"
|
|
if [ "$unit" = archied ]; then
|
|
systemctl --user "$action" "$unit.service"
|
|
return
|
|
fi
|
|
systemctl --user "$action" "$unit.service" || true
|
|
}
|
|
|
|
# start_units brings the deployment up in dependency order: the State Store
|
|
# before anything that dials it, the Gateway before archied and the UI.
|
|
start_units() {
|
|
local unit rc=0
|
|
for unit in $UNITS; do
|
|
echo "==> restarting $unit.service"
|
|
manage_unit restart "$unit" || rc=1
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# stop_units tears down in reverse, so consumers are gone before the services
|
|
# they depend on.
|
|
stop_units() {
|
|
local unit ordered="" rc=0
|
|
for unit in $UNITS; do
|
|
ordered="$unit${ordered:+ $ordered}"
|
|
done
|
|
for unit in $ordered; do
|
|
manage_unit stop "$unit" || rc=1
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
# rollback_gateway restores every binary the install replaced. A binary with
|
|
# no .prev was introduced by the failed release, so it is removed rather than
|
|
# left behind pointing at processes the restored release does not run.
|
|
rollback_gateway() {
|
|
local cmd rc=0
|
|
for cmd in $BINARIES; do
|
|
if [ -f "$BIN_DIR/$cmd.prev" ]; then
|
|
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
|
|
else
|
|
rm -f "$BIN_DIR/$cmd" || rc=1
|
|
fi
|
|
done
|
|
return $rc
|
|
}
|
|
|
|
rollback_agent_image() {
|
|
if [ ! -f "$AGENT_IMAGE_BACKUP" ]; then
|
|
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
|
return 1
|
|
fi
|
|
docker image load --input "$AGENT_IMAGE_BACKUP"
|
|
}
|
|
|
|
rollback_task_db() {
|
|
if [ -z "$TASK_DB_PATH" ]; then
|
|
echo "==> no task database path; cannot restore schema-compatible state" >&2
|
|
return 1
|
|
fi
|
|
if [ ! -f "$TASK_DB_BACKUP" ]; then
|
|
echo "==> no task database backup; cannot restore schema-compatible state" >&2
|
|
return 1
|
|
fi
|
|
sqlite3 "$TASK_DB_BACKUP" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1 || true
|
|
cp -f "$TASK_DB_BACKUP" "$TASK_DB_PATH"
|
|
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
|
|
}
|
|
|
|
start_units
|
|
|
|
if wait_healthy; then
|
|
echo "==> new version is healthy"
|
|
write_report "passed" false
|
|
exit 0
|
|
fi
|
|
|
|
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back changed components"
|
|
rolled_back=false
|
|
# Stop before touching either the executable or its database. The service
|
|
# uses Restart=on-failure in the supported unit, so copying rollback state
|
|
# while a crash-loop child is alive can race with SQLite and execute the new
|
|
# binary after its files have been replaced.
|
|
if stop_units; then
|
|
case "$COMPONENTS" in
|
|
daemon)
|
|
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
|
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
|
|
rolled_back=true
|
|
fi
|
|
else
|
|
echo "==> no backup for archied; cannot roll it back automatically" >&2
|
|
fi
|
|
;;
|
|
agent)
|
|
if [ -f "$AGENT_IMAGE_BACKUP" ]; then
|
|
rollback_agent_image && rolled_back=true
|
|
else
|
|
echo "==> no backup for managed worker image $AGENT_IMAGE; cannot roll it back automatically" >&2
|
|
fi
|
|
;;
|
|
daemon,agent)
|
|
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
|
# component and leave the other on the failed release.
|
|
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
|
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
|
|
rolled_back=true
|
|
fi
|
|
else
|
|
echo "==> not all changed-component backups exist; cannot roll back automatically" >&2
|
|
fi
|
|
;;
|
|
*)
|
|
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
|
;;
|
|
esac
|
|
else
|
|
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
|
|
fi
|
|
if [ "$rolled_back" = true ]; then
|
|
start_units
|
|
fi
|
|
|
|
write_report "failed" "$rolled_back"
|