feat(archied): recovery commands on PostgreSQL; fail-closed pre-update snapshot, no automatic schema rollback
Without -db, backup/restore/validate/rollback resolve database_url from the config and work on the whole PostgreSQL database. Restore claims every service role; rollback claims the State Store role; validate refuses a schema newer than the binary. The installer refuses a daemon update it cannot snapshot first; the watchdog never restores the database and prints the explicit offline restore instead.
This commit is contained in:
1 parent
b39f7c682f
commit
fe819d489a
11 files changed
+588
-48
No files matched your search
@@ -80,6 +80,7 @@ AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
||||
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
# The address the daemon dials for its state, so the watchdog can wait for it to
|
||||
# accept connections before it starts archied. Read with the same narrow awk as
|
||||
# the task database path and for the same reason: the daemon stays the authority
|
||||
@@ -284,6 +285,22 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
|
||||
fi
|
||||
fi
|
||||
|
||||
# The pre-migration snapshot of the PostgreSQL database: the whole database,
|
||||
# since every service shares it. The new State Store migrates on start and a
|
||||
# migration is never undone automatically (goose Down drops tables), so this
|
||||
# snapshot is the only way back, restored by an operator with every service
|
||||
# stopped. The recovery subcommand resolves database_url from the same config
|
||||
# the services boot with, so the connection string never passes through this
|
||||
# script. No snapshot, no update.
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> snapshotting the database before the update migrates it"
|
||||
mkdir -p "$(dirname "$DB_SNAPSHOT")"
|
||||
if ! go run ./cmd/archie-state-store backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"; then
|
||||
echo "refusing to install $GW: could not snapshot the database named by $CONFIG_PATH, so a failed migration could not be recovered" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Refresh adapters from the approved release source. The running shell
|
||||
# continues with this copy; subsequent updates use the current contract.
|
||||
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
|
||||
@@ -377,6 +394,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
|
||||
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
|
||||
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
|
||||
--setenv=ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" \
|
||||
--setenv=ARCHIE_CONFIG_PATH="$CONFIG_PATH" \
|
||||
--setenv=ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" \
|
||||
"$BIN_DIR/archie-update-watchdog"
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.versi
|
||||
AGENT_VERSION_PREVIOUS_EXISTS="${ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS:-0}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
||||
# The daemon exports its own health address (releaseupdate.CommandInstaller);
|
||||
# this default only covers a hand-driven run, and must stay the address archied
|
||||
# itself serves /healthz on -- [health] listen, defaulted in
|
||||
@@ -218,6 +220,16 @@ rollback_task_db() {
|
||||
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
|
||||
}
|
||||
|
||||
# explain_database_recovery states what a daemon rollback does not do. The
|
||||
# PostgreSQL database is never restored automatically: goose Down drops tables,
|
||||
# and a restore is destructive, loses every write after the snapshot and needs
|
||||
# every service stopped, so it is an operator's decision, not this script's.
|
||||
explain_database_recovery() {
|
||||
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
|
||||
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
|
||||
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
||||
}
|
||||
|
||||
start_units
|
||||
|
||||
if wait_healthy; then
|
||||
@@ -273,5 +285,8 @@ fi
|
||||
if [ "$rolled_back" = true ]; then
|
||||
start_units
|
||||
fi
|
||||
if [[ ",$COMPONENTS," == *,daemon,* ]]; then
|
||||
explain_database_recovery
|
||||
fi
|
||||
|
||||
write_report "failed" "$rolled_back"
|
||||
Reference in new issue
Block a user