fix(update): install and cycle every process in the release
A self-update to 1.23.0 failed its health check and rolled back. The adapter built and installed ./cmd/archied and nothing else, and the watchdog restarted archied.service and nothing else, so the new daemon came up against a State Store, Gateway and UI still on the previous release. Since 1.23.0 archied cannot boot without a reachable State Store (openStateStoreAdapter), which on a host that had never run one is an immediate exit, a failed health check, and a rollback that looks like a bad release rather than an incomplete deployment. Install every binary the reference deployment runs, versioned off the archied tag, and back each up so the rollback restores all of them together. A binary with no .prev was introduced by the failed release, so rollback removes it instead of leaving one behind that points at processes the restored release does not run. Restart in dependency order: the State Store owns archie.db and every other process dials it, the Gateway serves the contract archied and the UI dial, so each comes up before its consumers and stops after them. archied itself is cycled unconditionally and a failed stop still aborts the rollback, because a presence probe that guesses wrong must never skip the stop that keeps a live process off the binary and the database being restored. The siblings are best-effort, which is safe because the adapter now refuses the update outright, before touching the host, when a unit for one of them is missing, naming what to create. install.sh had the same single-binary assumption, so a fresh install produced a host that could not start either. Closes archie-core-hbqk.
This commit is contained in:
1 parent
2d228a00b4
commit
99afccc3b2
4 files changed
+285
-35
No files matched your search
+12
-3
@@ -162,7 +162,7 @@ fi
|
||||
# 4. Build and install the native daemon. archie-agent is deployed only as
|
||||
# the managed task image; installing a host binary would imply an unsupported
|
||||
# host execution path.
|
||||
echo "==> Building native archied..."
|
||||
echo "==> Building native archie binaries..."
|
||||
(
|
||||
cd "${SRC_DIR}"
|
||||
# installtype.buildType must be stamped here: an unstamped archied
|
||||
@@ -171,10 +171,19 @@ echo "==> Building native archied..."
|
||||
# even the right kind of update for a script-built native binary.
|
||||
GATEWAY_VERSION="$(git describe --tags --match 'archied/v*' 2>/dev/null | sed 's|^archied/v||' || echo dev)"
|
||||
RUNTIME_VERSION="$(git describe --tags --match 'archie/v*' 2>/dev/null | sed 's|^archie/v||' || echo dev)"
|
||||
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" -o "${ARCHIE_BIN_DIR}/archied" ./cmd/archied
|
||||
LDFLAGS="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION}"
|
||||
LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION}"
|
||||
LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
|
||||
# archied does not run alone: the State Store owns archie.db, the Gateway
|
||||
# serves the chat contract, and the dashboard is its own process. Building
|
||||
# only archied leaves it unable to boot. Keep this list and the one in
|
||||
# scripts/archie-update-install together.
|
||||
for cmd in archied archie-gateway archie-state-store archie-ui archie-playbooks; do
|
||||
go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}"
|
||||
done
|
||||
install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install"
|
||||
)
|
||||
echo " Installed archied and updater to ${ARCHIE_BIN_DIR}/"
|
||||
echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-playbooks and updater to ${ARCHIE_BIN_DIR}/"
|
||||
|
||||
# 5. Interactive Configuration: Forge & LLM Provider Setup
|
||||
if [ ! -f "${ENV_FILE}" ]; then
|
||||
|
||||
@@ -273,6 +273,15 @@ func createSQLiteDatabase(t *testing.T, path string, version int) {
|
||||
}
|
||||
|
||||
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
||||
t.Helper()
|
||||
result, calls, _ := runUpdateInstall(t, environment, false)
|
||||
return result, calls
|
||||
}
|
||||
|
||||
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
|
||||
// returns the parsed result (empty when the run was expected to fail), the
|
||||
// recorded calls, and the combined output.
|
||||
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -303,6 +312,9 @@ case "$*" in
|
||||
source_dir="${@: -1}"
|
||||
mkdir -p "$source_dir"
|
||||
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
|
||||
for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do
|
||||
mkdir -p "$source_dir/cmd/$c"
|
||||
done
|
||||
: > "$source_dir/docker-compose.yml"
|
||||
;;
|
||||
*"rev-parse --verify refs/tags/"*) echo approved-release-commit ;;
|
||||
@@ -331,6 +343,15 @@ if [ "$1" = image ] && [ "$2" = save ]; then
|
||||
fi
|
||||
`)
|
||||
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
|
||||
writeFakeCommand(t, fakeDir, "systemctl", `
|
||||
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
||||
if [ "$2" = list-unit-files ]; then
|
||||
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
|
||||
*" $3 "*) exit 0 ;;
|
||||
esac
|
||||
printf '%s enabled enabled\n' "$3"
|
||||
fi
|
||||
`)
|
||||
writeFakeCommand(t, fakeDir, "install", `
|
||||
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
|
||||
/usr/bin/install "$@"
|
||||
@@ -366,9 +387,23 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
switch {
|
||||
case wantErr && err == nil:
|
||||
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
|
||||
case !wantErr && err != nil:
|
||||
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
|
||||
}
|
||||
callsData, err := os.ReadFile(callsPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls []string
|
||||
if len(callsData) > 0 {
|
||||
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
||||
}
|
||||
if wantErr {
|
||||
return Result{}, calls, string(output)
|
||||
}
|
||||
var result Result
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
||||
found := false
|
||||
@@ -383,15 +418,7 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
||||
if !found {
|
||||
t.Fatalf("installer output has no result sentinel:\n%s", output)
|
||||
}
|
||||
callsData, err := os.ReadFile(callsPath)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var calls []string
|
||||
if len(callsData) > 0 {
|
||||
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
||||
}
|
||||
return result, calls
|
||||
return result, calls, string(output)
|
||||
}
|
||||
|
||||
func writeFakeCommand(t *testing.T, dir, name, body string) {
|
||||
@@ -520,3 +547,118 @@ func assertCallAbsent(t *testing.T, calls []string, fragment string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An archied release is five processes. Installing the daemon binary alone
|
||||
// left the State Store, Gateway and UI on the previous release, so archied
|
||||
// came up against a State Store nothing had started, failed its health check
|
||||
// and rolled back (archie-core-hbqk).
|
||||
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
|
||||
_, calls := runUpdateInstallScript(t, map[string]string{
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
||||
})
|
||||
|
||||
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} {
|
||||
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
|
||||
assertCallContains(t, calls, "install -m755", "/"+cmd)
|
||||
}
|
||||
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
||||
// The watchdog cannot restart what it is not told about.
|
||||
assertCallContains(t, calls, "systemd-run",
|
||||
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui")
|
||||
assertCallContains(t, calls, "systemd-run",
|
||||
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks")
|
||||
}
|
||||
|
||||
// Installing binaries for processes the host has no unit for produces exactly
|
||||
// the failure this bug was: archied dials a State Store nothing starts. Refuse
|
||||
// before touching the host instead, and say what is missing.
|
||||
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
|
||||
_, calls, output := runUpdateInstall(t, map[string]string{
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
||||
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
|
||||
}, true)
|
||||
|
||||
if !strings.Contains(output, "archie-state-store.service") {
|
||||
t.Errorf("refusal must name the missing unit; output = %q", output)
|
||||
}
|
||||
// Nothing may be replaced on a host it would leave broken.
|
||||
assertCallAbsent(t, calls, "go build")
|
||||
assertCallAbsent(t, calls, "install -m755")
|
||||
}
|
||||
|
||||
// The deployment is several processes, so the watchdog must cycle all of them,
|
||||
// and in dependency order: the State Store owns archie.db and everything dials
|
||||
// it, so it starts first and stops last (archie-core-hbqk).
|
||||
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
work := t.TempDir()
|
||||
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// archie-ui is new in this release: it has no .prev, so rolling back must
|
||||
// remove it rather than leave a binary from a release that was withdrawn.
|
||||
for name, content := range map[string]string{
|
||||
"archied": "new", "archied.prev": "old",
|
||||
"archie-state-store": "new", "archie-state-store.prev": "old",
|
||||
"archie-ui": "new",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
callsPath := filepath.Join(work, "calls")
|
||||
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
||||
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
||||
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
||||
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
||||
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
||||
"ARCHIE_UPDATE_COMPONENTS=daemon",
|
||||
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
|
||||
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
|
||||
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
||||
}
|
||||
calls, err := os.ReadFile(callsPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
indexOf := func(want string) int {
|
||||
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
||||
if line == want {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
startStore := indexOf("systemctl --user restart archie-state-store.service")
|
||||
startDaemon := indexOf("systemctl --user restart archied.service")
|
||||
stopUI := indexOf("systemctl --user stop archie-ui.service")
|
||||
stopStore := indexOf("systemctl --user stop archie-state-store.service")
|
||||
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
|
||||
t.Errorf("State Store must start before archied; calls =\n%s", calls)
|
||||
}
|
||||
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
|
||||
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
|
||||
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
|
||||
}
|
||||
for _, name := range []string{"archied", "archie-state-store"} {
|
||||
got, err := os.ReadFile(filepath.Join(binDir, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "old" {
|
||||
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,17 @@ AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
|
||||
# The archied release is a set of processes, not one binary. Listed in start
|
||||
# order: the State Store owns archie.db and every other process dials it, and
|
||||
# the Gateway serves the chat contract archied and the UI dial, so each comes
|
||||
# up before its consumers. archie-playbooks is a CLI, so it is installed but
|
||||
# never restarted. deployments/systemd-user-service.md is the reference
|
||||
# deployment; installing archied alone leaves the rest on the previous
|
||||
# release, which is why a 1.23.0 update failed its health check
|
||||
# (archie-core-hbqk).
|
||||
GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"
|
||||
GATEWAY_BINARIES="$GATEWAY_SERVICES archie-playbooks"
|
||||
|
||||
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
|
||||
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
|
||||
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
|
||||
@@ -133,6 +144,25 @@ esac
|
||||
git -C "$source_dir" checkout --quiet --detach "$build_commit"
|
||||
cd "$source_dir"
|
||||
|
||||
# Refuse before touching anything if the release runs processes this host has
|
||||
# no unit for. Installing the binaries anyway would leave archied dialling a
|
||||
# State Store that nothing starts: it fails its health check and rolls back,
|
||||
# which reads as a broken release rather than an incomplete deployment.
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
missing_units=""
|
||||
for svc in $GATEWAY_SERVICES; do
|
||||
[ -d "$source_dir/cmd/$svc" ] || continue
|
||||
if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then
|
||||
missing_units="${missing_units:+$missing_units }$svc.service"
|
||||
fi
|
||||
done
|
||||
if [ -n "$missing_units" ]; then
|
||||
echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2
|
||||
echo "create them from deployments/systemd-user-service.md, then retry the update" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Refresh adapters from the approved release source. The running shell
|
||||
# continues with this copy; subsequent updates use the current contract.
|
||||
install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install"
|
||||
@@ -145,16 +175,34 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_
|
||||
sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'"
|
||||
fi
|
||||
|
||||
# built accumulates only the binaries this release actually ships, so an
|
||||
# update to a release predating one of these processes stays buildable.
|
||||
built=""
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> building archied $GW"
|
||||
go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \
|
||||
-o "$tmp/archied" ./cmd/archied
|
||||
fi
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
echo "==> backing up current archied binary"
|
||||
[ -f "$BIN_DIR/archied" ] && cp -f "$BIN_DIR/archied" "$BIN_DIR/archied.prev"
|
||||
echo "==> installing archied"
|
||||
install -m755 "$tmp/archied" "$BIN_DIR/archied"
|
||||
ldflags="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW"
|
||||
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT"
|
||||
ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary"
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
[ -d "./cmd/$cmd" ] || continue
|
||||
echo "==> building $cmd $GW"
|
||||
go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd"
|
||||
built="${built:+$built }$cmd"
|
||||
done
|
||||
|
||||
echo "==> backing up current binaries"
|
||||
for cmd in $built; do
|
||||
if [ -f "$BIN_DIR/$cmd" ]; then
|
||||
cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev"
|
||||
else
|
||||
# New in this release: record that there was nothing here, so a
|
||||
# rollback removes it instead of leaving a stray binary behind.
|
||||
rm -f "$BIN_DIR/$cmd.prev"
|
||||
fi
|
||||
done
|
||||
echo "==> installing $built"
|
||||
for cmd in $built; do
|
||||
install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd"
|
||||
done
|
||||
fi
|
||||
if [[ ",$components," == *,agent,* ]]; then
|
||||
echo "==> saving managed worker image $AGENT_IMAGE for rollback"
|
||||
@@ -172,6 +220,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
|
||||
--setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \
|
||||
--setenv=ARCHIE_UPDATE_COMPONENTS="$components" \
|
||||
--setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \
|
||||
--setenv=ARCHIE_UPDATE_BINARIES="$built" \
|
||||
--setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \
|
||||
--setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \
|
||||
--setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \
|
||||
|
||||
@@ -22,6 +22,11 @@ REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
||||
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
||||
POLL_INTERVAL_SECONDS=3
|
||||
COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}"
|
||||
# Set by archie-update-install: the units to restart, in start order, and the
|
||||
# binaries it replaced. The defaults keep a hand-driven run on a pre-
|
||||
# decomposition host working.
|
||||
UNITS="${ARCHIE_UPDATE_UNITS:-archied}"
|
||||
BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}"
|
||||
|
||||
json_escape() {
|
||||
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
||||
@@ -72,13 +77,58 @@ wait_healthy() {
|
||||
return 1
|
||||
}
|
||||
|
||||
rollback_component() {
|
||||
local name="$1"
|
||||
if [ ! -f "$BIN_DIR/$name.prev" ]; then
|
||||
echo "==> no backup for $name; cannot roll it back automatically" >&2
|
||||
return 1
|
||||
# manage_unit runs one systemctl action. archied is required: this whole
|
||||
# mechanism exists to restart it, and a failed stop must abort the rollback
|
||||
# rather than let it race a live process over the binary and the database.
|
||||
# The sibling processes are best-effort, because archie-update-install has
|
||||
# already refused the update if a unit for one of them was missing, and a
|
||||
# probe that guesses wrong must not silently skip the required stop.
|
||||
manage_unit() {
|
||||
local action="$1" unit="$2"
|
||||
if [ "$unit" = archied ]; then
|
||||
systemctl --user "$action" "$unit.service"
|
||||
return
|
||||
fi
|
||||
cp -f "$BIN_DIR/$name.prev" "$BIN_DIR/$name"
|
||||
systemctl --user "$action" "$unit.service" || true
|
||||
}
|
||||
|
||||
# start_units brings the deployment up in dependency order: the State Store
|
||||
# before anything that dials it, the Gateway before archied and the UI.
|
||||
start_units() {
|
||||
local unit rc=0
|
||||
for unit in $UNITS; do
|
||||
echo "==> restarting $unit.service"
|
||||
manage_unit restart "$unit" || rc=1
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
# stop_units tears down in reverse, so consumers are gone before the services
|
||||
# they depend on.
|
||||
stop_units() {
|
||||
local unit ordered="" rc=0
|
||||
for unit in $UNITS; do
|
||||
ordered="$unit${ordered:+ $ordered}"
|
||||
done
|
||||
for unit in $ordered; do
|
||||
manage_unit stop "$unit" || rc=1
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
# rollback_gateway restores every binary the install replaced. A binary with
|
||||
# no .prev was introduced by the failed release, so it is removed rather than
|
||||
# left behind pointing at processes the restored release does not run.
|
||||
rollback_gateway() {
|
||||
local cmd rc=0
|
||||
for cmd in $BINARIES; do
|
||||
if [ -f "$BIN_DIR/$cmd.prev" ]; then
|
||||
cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1
|
||||
else
|
||||
rm -f "$BIN_DIR/$cmd" || rc=1
|
||||
fi
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
rollback_agent_image() {
|
||||
@@ -103,8 +153,7 @@ rollback_task_db() {
|
||||
rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm"
|
||||
}
|
||||
|
||||
echo "==> restarting archied.service"
|
||||
systemctl --user restart archied.service
|
||||
start_units
|
||||
|
||||
if wait_healthy; then
|
||||
echo "==> new version is healthy"
|
||||
@@ -118,11 +167,11 @@ rolled_back=false
|
||||
# uses Restart=on-failure in the supported unit, so copying rollback state
|
||||
# while a crash-loop child is alive can race with SQLite and execute the new
|
||||
# binary after its files have been replaced.
|
||||
if systemctl --user stop archied.service; then
|
||||
if stop_units; then
|
||||
case "$COMPONENTS" in
|
||||
daemon)
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
||||
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
|
||||
if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
@@ -140,7 +189,7 @@ if systemctl --user stop archied.service; then
|
||||
# Preserve the old all-or-nothing rollback guarantee: never restore one
|
||||
# component and leave the other on the failed release.
|
||||
if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then
|
||||
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then
|
||||
if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then
|
||||
rolled_back=true
|
||||
fi
|
||||
else
|
||||
@@ -152,10 +201,10 @@ if systemctl --user stop archied.service; then
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo "==> could not stop archied.service; refusing rollback while it may be running" >&2
|
||||
echo "==> could not stop every unit; refusing rollback while one may be running" >&2
|
||||
fi
|
||||
if [ "$rolled_back" = true ]; then
|
||||
systemctl --user restart archied.service
|
||||
start_units
|
||||
fi
|
||||
|
||||
write_report "failed" "$rolled_back"
|
||||
Reference in new issue
Block a user