fix(update): install and cycle every process in the release

A self-update to 1.23.0 failed its health check and rolled back. The
adapter built and installed ./cmd/archied and nothing else, and the
watchdog restarted archied.service and nothing else, so the new daemon
came up against a State Store, Gateway and UI still on the previous
release. Since 1.23.0 archied cannot boot without a reachable State
Store (openStateStoreAdapter), which on a host that had never run one is
an immediate exit, a failed health check, and a rollback that looks like
a bad release rather than an incomplete deployment.

Install every binary the reference deployment runs, versioned off the
archied tag, and back each up so the rollback restores all of them
together. A binary with no .prev was introduced by the failed release, so
rollback removes it instead of leaving one behind that points at
processes the restored release does not run.

Restart in dependency order: the State Store owns archie.db and every
other process dials it, the Gateway serves the contract archied and the
UI dial, so each comes up before its consumers and stops after them.
archied itself is cycled unconditionally and a failed stop still aborts
the rollback, because a presence probe that guesses wrong must never
skip the stop that keeps a live process off the binary and the database
being restored. The siblings are best-effort, which is safe because the
adapter now refuses the update outright, before touching the host, when
a unit for one of them is missing, naming what to create.

install.sh had the same single-binary assumption, so a fresh install
produced a host that could not start either.

Closes archie-core-hbqk.
This commit is contained in:
sam committed 2026-09-12 09:52:01 +10:00
1 parent 2d228a00b4
commit 99afccc3b2
4 files changed
+285 -35

No files matched your search

+12 -3
View File
@@ -162,7 +162,7 @@ fi
# 4. Build and install the native daemon. archie-agent is deployed only as
# the managed task image; installing a host binary would imply an unsupported
# host execution path.
echo "==> Building native archied..."
echo "==> Building native archie binaries..."
(
cd "${SRC_DIR}"
# installtype.buildType must be stamped here: an unstamped archied
@@ -171,10 +171,19 @@ echo "==> Building native archied..."
# even the right kind of update for a script-built native binary.
GATEWAY_VERSION="$(git describe --tags --match 'archied/v*' 2>/dev/null | sed 's|^archied/v||' || echo dev)"
RUNTIME_VERSION="$(git describe --tags --match 'archie/v*' 2>/dev/null | sed 's|^archie/v||' || echo dev)"
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" -o "${ARCHIE_BIN_DIR}/archied" ./cmd/archied
LDFLAGS="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION}"
LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION}"
LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
# archied does not run alone: the State Store owns archie.db, the Gateway
# serves the chat contract, and the dashboard is its own process. Building
# only archied leaves it unable to boot. Keep this list and the one in
# scripts/archie-update-install together.
for cmd in archied archie-gateway archie-state-store archie-ui archie-playbooks; do
go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}"
done
install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install"
)
echo " Installed archied and updater to ${ARCHIE_BIN_DIR}/"
echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-playbooks and updater to ${ARCHIE_BIN_DIR}/"
# 5. Interactive Configuration: Forge & LLM Provider Setup
if [ ! -f "${ENV_FILE}" ]; then
+152 -10
View File
@@ -273,6 +273,15 @@ func createSQLiteDatabase(t *testing.T, path string, version int) {
}
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
t.Helper()
result, calls, _ := runUpdateInstall(t, environment, false)
return result, calls
}
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
// returns the parsed result (empty when the run was expected to fail), the
// recorded calls, and the combined output.
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
t.Helper()
ctx := t.Context()
@@ -303,6 +312,9 @@ case "$*" in
source_dir="${@: -1}"
mkdir -p "$source_dir"
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do
mkdir -p "$source_dir/cmd/$c"
done
: > "$source_dir/docker-compose.yml"
;;
*"rev-parse --verify refs/tags/"*) echo approved-release-commit ;;
@@ -331,6 +343,15 @@ if [ "$1" = image ] && [ "$2" = save ]; then
fi
`)
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "systemctl", `
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
if [ "$2" = list-unit-files ]; then
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
*" $3 "*) exit 0 ;;
esac
printf '%s enabled enabled\n' "$3"
fi
`)
writeFakeCommand(t, fakeDir, "install", `
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/install "$@"
@@ -366,9 +387,23 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
cmd.Env = append(cmd.Env, key+"="+value)
}
output, err := cmd.CombinedOutput()
if err != nil {
switch {
case wantErr && err == nil:
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
case !wantErr && err != nil:
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
if wantErr {
return Result{}, calls, string(output)
}
var result Result
scanner := bufio.NewScanner(strings.NewReader(string(output)))
found := false
@@ -383,15 +418,7 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
if !found {
t.Fatalf("installer output has no result sentinel:\n%s", output)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
return result, calls
return result, calls, string(output)
}
func writeFakeCommand(t *testing.T, dir, name, body string) {
@@ -520,3 +547,118 @@ func assertCallAbsent(t *testing.T, calls []string, fragment string) {
}
}
}
// An archied release is five processes. Installing the daemon binary alone
// left the State Store, Gateway and UI on the previous release, so archied
// came up against a State Store nothing had started, failed its health check
// and rolled back (archie-core-hbqk).
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
})
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} {
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
assertCallContains(t, calls, "install -m755", "/"+cmd)
}
assertCallAbsent(t, calls, "./cmd/archie-agent")
// The watchdog cannot restart what it is not told about.
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui")
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks")
}
// Installing binaries for processes the host has no unit for produces exactly
// the failure this bug was: archied dials a State Store nothing starts. Refuse
// before touching the host instead, and say what is missing.
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
_, calls, output := runUpdateInstall(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
}, true)
if !strings.Contains(output, "archie-state-store.service") {
t.Errorf("refusal must name the missing unit; output = %q", output)
}
// Nothing may be replaced on a host it would leave broken.
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
}
// The deployment is several processes, so the watchdog must cycle all of them,
// and in dependency order: the State Store owns archie.db and everything dials
// it, so it starts first and stops last (archie-core-hbqk).
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
// archie-ui is new in this release: it has no .prev, so rolling back must
// remove it rather than leave a binary from a release that was withdrawn.
for name, content := range map[string]string{
"archied": "new", "archied.prev": "old",
"archie-state-store": "new", "archie-state-store.prev": "old",
"archie-ui": "new",
} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
"ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
indexOf := func(want string) int {
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
if line == want {
return i
}
}
return -1
}
startStore := indexOf("systemctl --user restart archie-state-store.service")
startDaemon := indexOf("systemctl --user restart archied.service")
stopUI := indexOf("systemctl --user stop archie-ui.service")
stopStore := indexOf("systemctl --user stop archie-state-store.service")
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
t.Errorf("State Store must start before archied; calls =\n%s", calls)
}
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
}
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
}
for _, name := range []string{"archied", "archie-state-store"} {
got, err := os.ReadFile(filepath.Join(binDir, name))
if err != nil {
t.Fatal(err)
}
if string(got) != "old" {
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
}
}
}
+59 -9
View File
@@ -62,6 +62,17 @@ AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
# The archied release is a set of processes, not one binary. Listed in start
# order: the State Store owns archie.db and every other process dials it, and
# the Gateway serves the chat contract archied and the UI dial, so each comes
# up before its consumers. archie-playbooks is a CLI, so it is installed but
# never restarted. deployments/systemd-user-service.md is the reference
# deployment; installing archied alone leaves the rest on the previous
# release, which is why a 1.23.0 update failed its health check
# (archie-core-hbqk).
GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"
GATEWAY_BINARIES="$GATEWAY_SERVICES archie-playbooks"
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
@@ -133,6 +144,25 @@ esac
git -C "$source_dir" checkout --quiet --detach "$build_commit"
cd "$source_dir"
# Refuse before touching anything if the release runs processes this host has
# no unit for. Installing the binaries anyway would leave archied dialling a
# State Store that nothing starts: it fails its health check and rolls back,
# which reads as a broken release rather than an incomplete deployment.
if [[ ",$components," == *,daemon,* ]]; then
missing_units=""
for svc in $GATEWAY_SERVICES; do
[ -d "$source_dir/cmd/$svc" ] || continue
if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
missing_units="${missing_units:+$missing_units }$svc.service"
fi
done
if [ -n "$missing_units" ]; then
echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2
echo "create them from deployments/systemd-user-service.md, then retry the update" >&2
exit 1
fi
fi
# Refresh adapters from the approved release source. The running shell
# continues with this copy; subsequent updates use the current contract.
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
@@ -145,16 +175,34 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"
fi
# built accumulates only the binaries this release actually ships, so an
# update to a release predating one of these processes stays buildable.
built=""
if [[ ",$components," == *,daemon,* ]]; then
echo "==> building archied $GW"
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \
-o "$tmp/archied" ./cmd/archied
fi
if [[ ",$components," == *,daemon,* ]]; then
echo "==> backing up current archied binary"
[ -f "$BIN_DIR/archied" ] && cp -f "$BIN_DIR/archied" "$BIN_DIR/archied.prev"
echo "==> installing archied"
install -m755 "$tmp/archied" "$BIN_DIR/archied"
ldflags="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW"
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT"
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
for cmd in $GATEWAY_BINARIES; do
[ -d "./cmd/$cmd" ] || continue
echo "==> building $cmd $GW"
go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd"
built="${built:+$built }$cmd"
done
echo "==> backing up current binaries"
for cmd in $built; do
if [ -f "$BIN_DIR/$cmd" ]; then
cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev"
else
# New in this release: record that there was nothing here, so a
# rollback removes it instead of leaving a stray binary behind.
rm -f "$BIN_DIR/$cmd.prev"
fi
done
echo "==> installing $built"
for cmd in $built; do
install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd"
done
fi
if [[ ",$components," == *,agent,* ]]; then
echo "==> saving managed worker image $AGENT_IMAGE for rollback"
@@ -172,6 +220,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
--setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \
--setenv=ARCHIE_UPDATE_COMPONENTS="$components" \
--setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \
--setenv=ARCHIE_UPDATE_BINARIES="$built" \
--setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \
--setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \
--setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \
+62 -13
View File
@@ -22,6 +22,11 @@ REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
POLL_INTERVAL_SECONDS=3
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
# Set by archie-update-install: the units to restart, in start order, and the
# binaries it replaced. The defaults keep a hand-driven run on a pre-
# decomposition host working.
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
json_escape() {
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
@@ -72,13 +77,58 @@ wait_healthy() {
return 1
}
rollback_component() {
local name="$1"
if [ ! -f "$BIN_DIR/$name.prev" ]; then
echo "==> no backup for $name; cannot roll it back automatically" >&2
return 1
# manage_unit runs one systemctl action. archied is required: this whole
# mechanism exists to restart it, and a failed stop must abort the rollback
# rather than let it race a live process over the binary and the database.
# The sibling processes are best-effort, because archie-update-install has
# already refused the update if a unit for one of them was missing, and a
# probe that guesses wrong must not silently skip the required stop.
manage_unit() {
local action="$1" unit="$2"
if [ "$unit" = archied ]; then
systemctl --user "$action" "$unit.service"
return
fi
cp -f "$BIN_DIR/$name.prev" "$BIN_DIR/$name"
systemctl --user "$action" "$unit.service" || true
}
# start_units brings the deployment up in dependency order: the State Store
# before anything that dials it, the Gateway before archied and the UI.
start_units() {
local unit rc=0
for unit in $UNITS; do
echo "==> restarting $unit.service"
manage_unit restart "$unit" || rc=1
done
return $rc
}
# stop_units tears down in reverse, so consumers are gone before the services
# they depend on.
stop_units() {
local unit ordered="" rc=0
for unit in $UNITS; do
ordered="$unit${ordered:+ $ordered}"
done
for unit in $ordered; do
manage_unit stop "$unit" || rc=1
done
return $rc
}
# rollback_gateway restores every binary the install replaced. A binary with
# no .prev was introduced by the failed release, so it is removed rather than
# left behind pointing at processes the restored release does not run.
rollback_gateway() {
local cmd rc=0
for cmd in $BINARIES; do
if [ -f "$BIN_DIR/$cmd.prev" ]; then
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
else
rm -f "$BIN_DIR/$cmd" || rc=1
fi
done
return $rc
}
rollback_agent_image() {
@@ -103,8 +153,7 @@ rollback_task_db() {
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
}
echo "==> restarting archied.service"
systemctl --user restart archied.service
start_units
if wait_healthy; then
echo "==> new version is healthy"
@@ -118,11 +167,11 @@ rolled_back=false
# uses Restart=on-failure in the supported unit, so copying rollback state
# while a crash-loop child is alive can race with SQLite and execute the new
# binary after its files have been replaced.
if systemctl --user stop archied.service; then
if stop_units; then
case "$COMPONENTS" in
daemon)
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
@@ -140,7 +189,7 @@ if systemctl --user stop archied.service; then
# Preserve the old all-or-nothing rollback guarantee: never restore one
# component and leave the other on the failed release.
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
rolled_back=true
fi
else
@@ -152,10 +201,10 @@ if systemctl --user stop archied.service; then
;;
esac
else
echo "==> could not stop archied.service; refusing rollback while it may be running" >&2
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
fi
if [ "$rolled_back" = true ]; then
systemctl --user restart archied.service
start_units
fi
write_report "failed" "$rolled_back"