The /update command went silent between "starting" and one final message, gave no version summary, and required manual intervention on failure. CommandInstaller now streams the installer script's stdout live instead of buffering it, and parses a structured result (previous/installed versions, per-component status) from a sentinel line. Because the reference install script hands the daemon restart off to a detached systemd unit specifically so restarting archied.service doesn't kill the process that requested the update, the outcome is reported in two phases: the synchronous build/install result comes back through Install() itself, and the post-restart health check/rollback outcome is written by a new watchdog script and relayed by whichever archied process boots next, via a small JSON handoff file cleared once relayed. Also fixes three defects an adversarial review pass found in the first draft: the caller's request/handler context was passed straight into the install child process, so a concurrent /restart or a browser disconnect could SIGKILL it mid binary-copy; Telegram and the web dashboard each tracked "install in progress" with their own private flag despite sharing one Service, allowing two installs to race the same .prev backups; and the watchdog reported rolled_back:true even when no backup existed to restore, which would have told an operator to stand down while the daemon was still broken. internal/webui gains an unauthenticated /healthz the watchdog polls after restarting archied.
77 lines
3.2 KiB
Bash
Executable File
77 lines
3.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# archie-update-watchdog -- restarts archied after an update, verifies the
|
|
# new version comes up healthy, and rolls back automatically if it doesn't.
|
|
#
|
|
# Launched detached (via systemd-run) by archie-update-install, after that
|
|
# script's own synchronous build/install work already completed and
|
|
# reported success back to chat. This script owns everything that happens
|
|
# after that point: nothing here can be observed by the Go process that
|
|
# requested the update, because restarting archied.service kills it.
|
|
#
|
|
# This is deliberately the ONLY place a crash-loop on the newly installed
|
|
# binary gets detected and rolled back -- by definition, no archied Go code
|
|
# can be running to do that itself if the new binary never gets that far.
|
|
set -uo pipefail
|
|
|
|
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
|
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}"
|
|
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
|
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
|
POLL_INTERVAL_SECONDS=3
|
|
|
|
# json_escape minimally escapes a string for embedding in a JSON document
|
|
# built by shell interpolation: backslashes first (so the quote-escaping
|
|
# backslash it inserts next isn't itself re-escaped), then double quotes.
|
|
json_escape() {
|
|
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
write_report() {
|
|
local health_check="$1" rolled_back="$2"
|
|
[ -z "$REPORT_PATH" ] && return 0
|
|
mkdir -p "$(dirname "$REPORT_PATH")"
|
|
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
|
|
cat >"$tmp" <<EOF
|
|
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":{"daemon":"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")","agent":"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")"},"installed":{"daemon":"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")","agent":"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")"},"health_check":"$health_check","rolled_back":$rolled_back}
|
|
EOF
|
|
chmod 600 "$tmp"
|
|
mv -f "$tmp" "$REPORT_PATH"
|
|
}
|
|
|
|
wait_healthy() {
|
|
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
|
while [ "$SECONDS" -lt "$deadline" ]; do
|
|
if curl -sf -o /dev/null "$HEALTH_URL/healthz"; then
|
|
return 0
|
|
fi
|
|
sleep "$POLL_INTERVAL_SECONDS"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
echo "==> restarting archied.service"
|
|
systemctl --user restart archied.service
|
|
|
|
if wait_healthy; then
|
|
echo "==> new version is healthy"
|
|
write_report "passed" false
|
|
exit 0
|
|
fi
|
|
|
|
echo "==> health check failed after ${HEALTH_TIMEOUT_SECONDS}s; rolling back"
|
|
rolled_back=false
|
|
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$BIN_DIR/archie-agent.prev" ]; then
|
|
cp -f "$BIN_DIR/archied.prev" "$BIN_DIR/archied"
|
|
cp -f "$BIN_DIR/archie-agent.prev" "$BIN_DIR/archie-agent"
|
|
rolled_back=true
|
|
systemctl --user restart archied.service
|
|
else
|
|
echo "==> no backup binaries found; cannot roll back automatically" >&2
|
|
fi
|
|
|
|
# rolled_back reflects what actually happened, not what was attempted: a
|
|
# report claiming a healed system when no backup existed to restore from
|
|
# would tell an operator to stand down while the broken version is still
|
|
# what's running (or still failing to start at all).
|
|
write_report "failed" "$rolled_back"
|