archie-state-store gains the offline recovery commands
docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
asks for. They work on the task database file directly -- <db_path>-tasks.sqlite,
not db_path -- and need neither archied nor the Web UI, which matters because
the failure they exist for is the one that stops archied starting:
backup -db F -out F consistent snapshot (VACUUM INTO), verified as a
usable store before it is published; safe against
a serving store
restore -db F -from F replace the database with a snapshot, dropping the
replaced file's -wal/-shm
validate -db F readable, not corrupt, schema no newer than this
binary, every stored resource decoding under the
definition that owns it
rollback -db F -kind K replay an earlier revision of a stored resource
[-revision N] through the ordinary replace, recorded as a new
revision
No rollback RPC is added and none is needed: a revision carries its own value,
so restoring one is the same replace the dashboard and the channels already
use, by hand, with the store stopped.
"The State Store is stopped" is checked rather than assumed. The serving
process now takes an exclusive flock on <db>.lock for its whole life
(store.AcquireOwnership), and restore/rollback refuse with
`task database is owned by another process` while it holds it. backup is the
deliberate exception: the update installer runs inside the process it is
replacing and cannot stop the State Store.
scripts/archie-update-install takes its pre-flight snapshot through
`go run ./cmd/archie-state-store backup` from the release being installed
instead of its own sqlite3 shell, so the installer and an operator run one
implementation. The watchdog's refusal semantics are unchanged: an update that
cannot back up still must not start.
Docs: safe-change-and-recovery.md's "Recovery: not yet built" now describes
what exists.
306 lines
14 KiB
Bash
Executable File
306 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# archie-update-install -- component-aware update installation adapter.
|
|
#
|
|
# CommandInstaller passes the freshly revalidated, approved snapshot through
|
|
# ARCHIE_UPDATE_{DAEMON,AGENT}_{PREVIOUS,VERSION}. A VERSION is non-empty only
|
|
# when that component changed in the approved snapshot. This script must never
|
|
# rediscover a broader plan after approval.
|
|
set -euo pipefail
|
|
|
|
json_escape() {
|
|
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
|
}
|
|
|
|
SOURCE_URL="${ARCHIE_SOURCE_URL:-https://github.com/samcharles93/archie-core.git}"
|
|
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
|
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
|
export PATH="/usr/local/go/bin:$PATH"
|
|
|
|
# The worker image is daemon configuration, not an updater-specific setting.
|
|
# Honour an explicit environment override for nonstandard deployers; otherwise
|
|
# read the [containers] image from the same config archied starts with. This
|
|
# deliberately understands only that one scalar, rather than trying to parse
|
|
# general TOML in shell. The daemon remains the authority for validation.
|
|
configured_agent_image() {
|
|
local image
|
|
image="$(awk '
|
|
/^\[containers\][[:space:]]*(#.*)?$/ { in_containers = 1; next }
|
|
/^\[/ { in_containers = 0 }
|
|
in_containers && /^[[:space:]]*image[[:space:]]*=/ {
|
|
value = $0
|
|
sub(/^[[:space:]]*image[[:space:]]*=[[:space:]]*/, "", value)
|
|
sub(/[[:space:]]*(#.*)?$/, "", value)
|
|
if (value ~ /^".*"$/) {
|
|
sub(/^"/, "", value)
|
|
sub(/"$/, "", value)
|
|
print value
|
|
} else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) {
|
|
print substr(value, 2, length(value) - 2)
|
|
}
|
|
exit
|
|
}
|
|
' "$CONFIG_PATH" 2>/dev/null || true)"
|
|
printf '%s' "$image"
|
|
}
|
|
|
|
# The task store is `<db_path>-tasks.sqlite`, not db_path itself: the daemon
|
|
# derives it with taskDBPath() in internal/app/archied/main.go, and the
|
|
# configured path may be a zero-byte placeholder while the real store sits
|
|
# beside it. Backing up the literal value would produce an empty backup, and a
|
|
# rollback would then restore that empty file over the database and delete its
|
|
# -wal/-shm -- destroying the task store it exists to protect.
|
|
#
|
|
# Only the top-level key counts. db_path also appears in other sections
|
|
# ([indexing] owns its own database), and the first match in the file can
|
|
# easily be one of those.
|
|
configured_task_db_path() {
|
|
local base
|
|
base="$(awk '
|
|
BEGIN { top = 1 }
|
|
/^[[:space:]]*\[/ { top = 0 }
|
|
top && /^[[:space:]]*db_path[[:space:]]*=/ {
|
|
value = $0
|
|
sub(/^[[:space:]]*db_path[[:space:]]*=[[:space:]]*/, "", value)
|
|
sub(/[[:space:]]*(#.*)?$/, "", value)
|
|
if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value }
|
|
else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2)
|
|
exit
|
|
}
|
|
' "$CONFIG_PATH" 2>/dev/null || true)"
|
|
[ -n "$base" ] || return 0
|
|
printf '%s' "$base-tasks.sqlite"
|
|
}
|
|
|
|
AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-$(configured_agent_image)}"
|
|
AGENT_IMAGE="${AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
|
AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
|
|
# State for the managed image, independent of archied's daemon build stamp.
|
|
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
|
AGENT_VERSION_BACKUP="${ARCHIE_AGENT_VERSION_BACKUP:-$BIN_DIR/archie-agent.version.prev}"
|
|
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
|
|
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
|
|
|
# The archied release is a set of processes, not one binary. Listed in start
|
|
# order: the State Store owns archie.db and every other process dials it, and
|
|
# the Gateway serves the chat contract archied and the UI dial, so each comes
|
|
# up before its consumers. archie-playbooks is a CLI, so it is installed but
|
|
# never restarted. deployments/systemd-user-service.md is the reference
|
|
# deployment; installing archied alone leaves the rest on the previous
|
|
# release, which is why a 1.23.0 update failed its health check
|
|
# (archie-core-hbqk).
|
|
GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"
|
|
GATEWAY_BINARIES="$GATEWAY_SERVICES archie-playbooks"
|
|
|
|
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
|
|
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
|
|
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
|
|
RT_TARGET="${ARCHIE_UPDATE_AGENT_VERSION:-}"
|
|
GW="$PREV_GW"
|
|
RT="$PREV_RT"
|
|
components=""
|
|
|
|
if [ -n "$GW_TARGET" ] && [ "$GW_TARGET" != "$PREV_GW" ]; then
|
|
GW="$GW_TARGET"
|
|
components="daemon"
|
|
fi
|
|
if [ -n "$RT_TARGET" ] && [ "$RT_TARGET" != "$PREV_RT" ]; then
|
|
case "$RT_TARGET" in
|
|
dev|unknown) echo "refusing to build a released runtime image with version '$RT_TARGET'" >&2; exit 1 ;;
|
|
esac
|
|
RT="$RT_TARGET"
|
|
components="${components:+$components,}agent"
|
|
fi
|
|
|
|
if [ -z "$components" ]; then
|
|
echo '==> no approved component changes; nothing to install'
|
|
echo 'ARCHIE_UPDATE_RESULT {"previous":{},"installed":{},"components":[],"restart_requested":false}'
|
|
exit 0
|
|
fi
|
|
|
|
valid_release_version() {
|
|
[[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$ ]]
|
|
}
|
|
|
|
for target in "$GW_TARGET" "$RT_TARGET"; do
|
|
if [ -n "$target" ] && ! valid_release_version "$target"; then
|
|
echo "refusing non-release component version '$target'" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
tmp="$(mktemp -d)"
|
|
source_dir="$tmp/source"
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
|
|
echo "==> fetching approved component releases"
|
|
# Updates are independent of the agent/operator checkout. Fetch into an
|
|
# invocation-scoped clone, resolve the approved tags there, and build the
|
|
# installed binary from that isolated source tree.
|
|
git clone --quiet --filter=blob:none --no-checkout "$SOURCE_URL" "$source_dir"
|
|
git -C "$source_dir" fetch --quiet --tags origin
|
|
|
|
# Build from the exact approved release commit, not the moving branch tip.
|
|
# Resolve each changed component's tag commit independently: components are
|
|
# independently versioned (RELEASING.md), so the daemon and agent tags usually
|
|
# sit on different commits -- most releases are gateway-only. The first
|
|
# checkout is the daemon release when one is being installed, because the
|
|
# adapter refresh below reads the scripts of the release being installed; the
|
|
# agent image is rebuilt from its own tag's commit further down.
|
|
gw_commit=""
|
|
rt_commit=""
|
|
if [[ ",$components," == *,daemon,* ]]; then
|
|
gw_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archied/v$GW^{commit}")"
|
|
fi
|
|
if [[ ",$components," == *,agent,* ]]; then
|
|
rt_commit="$(git -C "$source_dir" rev-parse --verify "refs/tags/archie/v$RT^{commit}")"
|
|
fi
|
|
build_commit="$gw_commit"
|
|
[ -n "$build_commit" ] || build_commit="$rt_commit"
|
|
git -C "$source_dir" checkout --quiet --detach "$build_commit"
|
|
cd "$source_dir"
|
|
|
|
# Refuse before touching anything if the release runs processes this host has
|
|
# no unit for. Installing the binaries anyway would leave archied dialling a
|
|
# State Store that nothing starts: it fails its health check and rolls back,
|
|
# which reads as a broken release rather than an incomplete deployment.
|
|
if [[ ",$components," == *,daemon,* ]]; then
|
|
missing_units=""
|
|
for svc in $GATEWAY_SERVICES; do
|
|
[ -d "$source_dir/cmd/$svc" ] || continue
|
|
if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
|
|
missing_units="${missing_units:+$missing_units }$svc.service"
|
|
fi
|
|
done
|
|
if [ -n "$missing_units" ]; then
|
|
echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2
|
|
echo "create them from deployments/systemd-user-service.md, then retry the update" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Pre-flight, before any write to the host. The task database backup is the
|
|
# watchdog's only route back from a forward schema migration, so an update
|
|
# that cannot take one must not start: once binaries are replaced, a failure
|
|
# leaves the previous release unable to read the store it just migrated. This
|
|
# runs ahead of the adapter refresh below so a refusal changes nothing at all.
|
|
#
|
|
# The snapshot is taken by the recovery subcommand of the release being
|
|
# installed, from its own checkout, rather than by hand-rolled sqlite3 shell:
|
|
# the operator running `archie-state-store backup` by hand and the installer
|
|
# are then the same implementation, and the two cannot drift. The installed
|
|
# binary is the previous release, which may predate the subcommand, so it
|
|
# cannot be the one asked -- and a release whose recovery path will not build is a
|
|
# release that cannot back up, which is exactly what this pre-flight refuses.
|
|
if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_DB_PATH" ]; then
|
|
echo "==> backing up task database for schema-safe rollback"
|
|
mkdir -p "$(dirname "$TASK_DB_BACKUP")"
|
|
if ! go run ./cmd/archie-state-store backup -db "$TASK_DB_PATH" -out "$TASK_DB_BACKUP"; then
|
|
echo "refusing to install $GW: could not back up the task database at $TASK_DB_PATH, so a failed update could not be rolled back" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Refresh adapters from the approved release source. The running shell
|
|
# continues with this copy; subsequent updates use the current contract.
|
|
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
|
|
install -m755 "$source_dir/scripts/archie-update-watchdog" "$BIN_DIR/archie-update-watchdog"
|
|
install -m755 "$source_dir/scripts/archie-update-check" "$BIN_DIR/archie-update-check"
|
|
|
|
# built accumulates only the binaries this release actually ships, so an
|
|
# update to a release predating one of these processes stays buildable.
|
|
built=""
|
|
if [[ ",$components," == *,daemon,* ]]; then
|
|
ldflags="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW"
|
|
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT"
|
|
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
|
|
for cmd in $GATEWAY_BINARIES; do
|
|
[ -d "./cmd/$cmd" ] || continue
|
|
echo "==> building $cmd $GW"
|
|
go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd"
|
|
built="${built:+$built }$cmd"
|
|
done
|
|
|
|
echo "==> backing up current binaries"
|
|
for cmd in $built; do
|
|
if [ -f "$BIN_DIR/$cmd" ]; then
|
|
cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev"
|
|
else
|
|
# New in this release: record that there was nothing here, so a
|
|
# rollback removes it instead of leaving a stray binary behind.
|
|
rm -f "$BIN_DIR/$cmd.prev"
|
|
fi
|
|
done
|
|
echo "==> installing $built"
|
|
for cmd in $built; do
|
|
install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd"
|
|
done
|
|
fi
|
|
if [[ ",$components," == *,agent,* ]]; then
|
|
# Independent versioning: the agent tag usually points at a different
|
|
# commit than the daemon's, so switch back to it for the image build (a
|
|
# no-op checkout for an agent-only update, which never left this commit).
|
|
git checkout --quiet --detach "$rt_commit"
|
|
echo "==> saving managed worker image $AGENT_IMAGE for rollback"
|
|
docker image save --output "$AGENT_IMAGE_BACKUP" "$AGENT_IMAGE"
|
|
# Save explicit sidecar state, including the first-install (absent) case,
|
|
# before changing it. The watchdog restores this alongside the image.
|
|
if [ -f "$AGENT_VERSION_FILE" ]; then
|
|
cp -f "$AGENT_VERSION_FILE" "$AGENT_VERSION_BACKUP"
|
|
AGENT_VERSION_PREVIOUS_EXISTS=1
|
|
else
|
|
rm -f "$AGENT_VERSION_BACKUP"
|
|
AGENT_VERSION_PREVIOUS_EXISTS=0
|
|
fi
|
|
echo "==> rebuilding managed worker image $AGENT_IMAGE for runtime $RT"
|
|
docker build --build-arg "RUNTIME_VERSION=$RT" --tag "$AGENT_IMAGE" .
|
|
# Only record a version after the image build succeeds. Write beside the
|
|
# state atomically so the check never observes a partial version.
|
|
version_tmp="$(mktemp "$AGENT_VERSION_FILE.tmp.XXXXXX")"
|
|
printf '%s\n' "$RT" > "$version_tmp"
|
|
chmod 600 "$version_tmp"
|
|
mv -f "$version_tmp" "$AGENT_VERSION_FILE"
|
|
fi
|
|
|
|
echo "==> handing off to the update watchdog for restart and health verification"
|
|
systemd-run --user --unit=archied-update-restart --collect --quiet \
|
|
--setenv=ARCHIE_BIN_DIR="$BIN_DIR" \
|
|
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \
|
|
--setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \
|
|
--setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \
|
|
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
|
|
--setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \
|
|
--setenv=ARCHIE_UPDATE_COMPONENTS="$components" \
|
|
--setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \
|
|
--setenv=ARCHIE_UPDATE_BINARIES="$built" \
|
|
--setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \
|
|
--setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \
|
|
--setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \
|
|
--setenv=ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" \
|
|
--setenv=ARCHIE_AGENT_IMAGE="$AGENT_IMAGE" \
|
|
--setenv=ARCHIE_AGENT_IMAGE_BACKUP="$AGENT_IMAGE_BACKUP" \
|
|
--setenv=ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" \
|
|
--setenv=ARCHIE_AGENT_VERSION_BACKUP="$AGENT_VERSION_BACKUP" \
|
|
--setenv=ARCHIE_AGENT_VERSION_PREVIOUS_EXISTS="${AGENT_VERSION_PREVIOUS_EXISTS:-0}" \
|
|
--setenv=ARCHIE_TASK_DB_PATH="$TASK_DB_PATH" \
|
|
--setenv=ARCHIE_TASK_DB_BACKUP="$TASK_DB_BACKUP" \
|
|
"$BIN_DIR/archie-update-watchdog"
|
|
|
|
case "$components" in
|
|
daemon)
|
|
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}"
|
|
installed="{\"daemon\":\"$(json_escape "$GW")\"}"
|
|
results='[{"id":"daemon","label":"daemon","status":"updated"}]'
|
|
;;
|
|
agent)
|
|
previous="{\"agent\":\"$(json_escape "$PREV_RT")\"}"
|
|
installed="{\"agent\":\"$(json_escape "$RT")\"}"
|
|
results='[{"id":"agent","label":"agent","status":"updated"}]'
|
|
;;
|
|
daemon,agent)
|
|
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\",\"agent\":\"$(json_escape "$PREV_RT")\"}"
|
|
installed="{\"daemon\":\"$(json_escape "$GW")\",\"agent\":\"$(json_escape "$RT")\"}"
|
|
results='[{"id":"daemon","label":"daemon","status":"updated"},{"id":"agent","label":"agent","status":"updated"}]'
|
|
;;
|
|
esac
|
|
echo "ARCHIE_UPDATE_RESULT {\"previous\":$previous,\"installed\":$installed,\"components\":$results,\"restart_requested\":true}"
|