feat(archied): recovery commands on PostgreSQL; fail-closed pre-update snapshot, no automatic schema rollback

Without -db, backup/restore/validate/rollback resolve database_url from the
config and work on the whole PostgreSQL database. Restore claims every service
role; rollback claims the State Store role; validate refuses a schema newer
than the binary. The installer refuses a daemon update it cannot snapshot
first; the watchdog never restores the database and prints the explicit
offline restore instead.
This commit is contained in:
sam committed 2026-09-24 09:50:10 +10:00
1 parent b39f7c682f
commit fe819d489a
11 files changed
+588 -48

No files matched your search

+19
View File
@@ -80,6 +80,7 @@ AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
# The address the daemon dials for its state, so the watchdog can wait for it to
# accept connections before it starts archied. Read with the same narrow awk as
# the task database path and for the same reason: the daemon stays the authority
@@ -284,6 +285,22 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
fi
fi
# The pre-migration snapshot of the PostgreSQL database: the whole database,
# since every service shares it. The new State Store migrates on start and a
# migration is never undone automatically (goose Down drops tables), so this
# snapshot is the only way back, restored by an operator with every service
# stopped. The recovery subcommand resolves database_url from the same config
# the services boot with, so the connection string never passes through this
# script. No snapshot, no update.
if [[ ",$components," == *,daemon,* ]]; then
echo "==> snapshotting the database before the update migrates it"
mkdir -p "$(dirname "$DB_SNAPSHOT")"
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
exit 1
fi
fi
# Refresh adapters from the approved release source. The running shell
# continues with this copy; subsequent updates use the current contract.
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
@@ -377,6 +394,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
"$BIN_DIR/archie-update-watchdog"
+15
View File
@@ -15,6 +15,8 @@ AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.versi
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
# The daemon exports its own health address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# itself serves /healthz on -- [health] listen, defaulted in
@@ -218,6 +220,16 @@ rollback_task_db() {
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
# explain_database_recovery states what a daemon rollback does not do. The
# PostgreSQL database is never restored automatically: goose Down drops tables,
# and a restore is destructive, loses every write after the snapshot and needs
# every service stopped, so it is an operator's decision, not this script's.
explain_database_recovery() {
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
}
start_units
if wait_healthy; then
@@ -273,5 +285,8 @@ fi
if [ "$rolled_back" = true ]; then
start_units
fi
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
explain_database_recovery
fi
write_report "failed" "$rolled_back"