diff --git a/install.sh b/install.sh index f57b8aa4d..6722895a5 100755 --- a/install.sh +++ b/install.sh @@ -162,7 +162,7 @@ fi # 4. Build and install the native daemon. archie-agent is deployed only as # the managed task image; installing a host binary would imply an unsupported # host execution path. -echo "==> Building native archied..." +echo "==> Building native archie binaries..." ( cd "${SRC_DIR}" # installtype.buildType must be stamped here: an unstamped archied @@ -171,10 +171,19 @@ echo "==> Building native archied..." # even the right kind of update for a script-built native binary. GATEWAY_VERSION="$(git describe --tags --match 'archied/v*' 2>/dev/null | sed 's|^archied/v||' || echo dev)" RUNTIME_VERSION="$(git describe --tags --match 'archie/v*' 2>/dev/null | sed 's|^archie/v||' || echo dev)" - go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" -o "${ARCHIE_BIN_DIR}/archied" ./cmd/archied + LDFLAGS="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=${GATEWAY_VERSION}" + LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=${RUNTIME_VERSION}" + LDFLAGS="${LDFLAGS} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" + # archied does not run alone: the State Store owns archie.db, the Gateway + # serves the chat contract, and the dashboard is its own process. Building + # only archied leaves it unable to boot. Keep this list and the one in + # scripts/archie-update-install together. + for cmd in archied archie-gateway archie-state-store archie-ui archie-playbooks; do + go build -ldflags "${LDFLAGS}" -o "${ARCHIE_BIN_DIR}/${cmd}" "./cmd/${cmd}" + done install -m755 "${SRC_DIR}/scripts/archie-update-install" "${ARCHIE_BIN_DIR}/archie-update-install" ) -echo " Installed archied and updater to ${ARCHIE_BIN_DIR}/" +echo " Installed archied, archie-gateway, archie-state-store, archie-ui, archie-playbooks and updater to ${ARCHIE_BIN_DIR}/" # 5. Interactive Configuration: Forge & LLM Provider Setup if [ ! -f "${ENV_FILE}" ]; then diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index e20f58ea3..03e8bb24f 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -273,6 +273,15 @@ func createSQLiteDatabase(t *testing.T, path string, version int) { } func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) { + t.Helper() + result, calls, _ := runUpdateInstall(t, environment, false) + return result, calls +} + +// runUpdateInstall runs the adapter, optionally expecting it to refuse. It +// returns the parsed result (empty when the run was expected to fail), the +// recorded calls, and the combined output. +func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) { t.Helper() ctx := t.Context() @@ -303,6 +312,9 @@ case "$*" in source_dir="${@: -1}" mkdir -p "$source_dir" /usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts" + for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do + mkdir -p "$source_dir/cmd/$c" + done : > "$source_dir/docker-compose.yml" ;; *"rev-parse --verify refs/tags/"*) echo approved-release-commit ;; @@ -331,6 +343,15 @@ if [ "$1" = image ] && [ "$2" = save ]; then fi `) writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`) + writeFakeCommand(t, fakeDir, "systemctl", ` +printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS" +if [ "$2" = list-unit-files ]; then + case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in + *" $3 "*) exit 0 ;; + esac + printf '%s enabled enabled\n' "$3" +fi +`) writeFakeCommand(t, fakeDir, "install", ` printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS" /usr/bin/install "$@" @@ -366,9 +387,23 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS" cmd.Env = append(cmd.Env, key+"="+value) } output, err := cmd.CombinedOutput() - if err != nil { + switch { + case wantErr && err == nil: + t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output) + case !wantErr && err != nil: t.Fatalf("archie-update-install failed: %v\n%s", err, output) } + callsData, err := os.ReadFile(callsPath) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var calls []string + if len(callsData) > 0 { + calls = strings.Split(strings.TrimSpace(string(callsData)), "\n") + } + if wantErr { + return Result{}, calls, string(output) + } var result Result scanner := bufio.NewScanner(strings.NewReader(string(output))) found := false @@ -383,15 +418,7 @@ printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS" if !found { t.Fatalf("installer output has no result sentinel:\n%s", output) } - callsData, err := os.ReadFile(callsPath) - if err != nil && !os.IsNotExist(err) { - t.Fatal(err) - } - var calls []string - if len(callsData) > 0 { - calls = strings.Split(strings.TrimSpace(string(callsData)), "\n") - } - return result, calls + return result, calls, string(output) } func writeFakeCommand(t *testing.T, dir, name, body string) { @@ -520,3 +547,118 @@ func assertCallAbsent(t *testing.T, calls []string, fragment string) { } } } + +// An archied release is five processes. Installing the daemon binary alone +// left the State Store, Gateway and UI on the previous release, so archied +// came up against a State Store nothing had started, failed its health check +// and rolled back (archie-core-hbqk). +func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) { + _, calls := runUpdateInstallScript(t, map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", + "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", + }) + + for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} { + assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd) + assertCallContains(t, calls, "install -m755", "/"+cmd) + } + assertCallAbsent(t, calls, "./cmd/archie-agent") + // The watchdog cannot restart what it is not told about. + assertCallContains(t, calls, "systemd-run", + "--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui") + assertCallContains(t, calls, "systemd-run", + "--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks") +} + +// Installing binaries for processes the host has no unit for produces exactly +// the failure this bug was: archied dials a State Store nothing starts. Refuse +// before touching the host instead, and say what is missing. +func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) { + _, calls, output := runUpdateInstall(t, map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0", + "ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0", + "ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service", + }, true) + + if !strings.Contains(output, "archie-state-store.service") { + t.Errorf("refusal must name the missing unit; output = %q", output) + } + // Nothing may be replaced on a host it would leave broken. + assertCallAbsent(t, calls, "go build") + assertCallAbsent(t, calls, "install -m755") +} + +// The deployment is several processes, so the watchdog must cycle all of them, +// and in dependency order: the State Store owns archie.db and everything dials +// it, so it starts first and stops last (archie-core-hbqk). +func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + work := t.TempDir() + binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatal(err) + } + // archie-ui is new in this release: it has no .prev, so rolling back must + // remove it rather than leave a binary from a release that was withdrawn. + for name, content := range map[string]string{ + "archied": "new", "archied.prev": "old", + "archie-state-store": "new", "archie-state-store.prev": "old", + "archie-ui": "new", + } { + if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil { + t.Fatal(err) + } + } + callsPath := filepath.Join(work, "calls") + writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`) + writeFakeCommand(t, fakeDir, "curl", `exit 1`) + cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog")) + cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir, + "ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", + "ARCHIE_UPDATE_COMPONENTS=daemon", + "ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui", + "ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui", + "ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("watchdog failed: %v\n%s", err, output) + } + calls, err := os.ReadFile(callsPath) + if err != nil { + t.Fatal(err) + } + indexOf := func(want string) int { + for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") { + if line == want { + return i + } + } + return -1 + } + startStore := indexOf("systemctl --user restart archie-state-store.service") + startDaemon := indexOf("systemctl --user restart archied.service") + stopUI := indexOf("systemctl --user stop archie-ui.service") + stopStore := indexOf("systemctl --user stop archie-state-store.service") + if startStore < 0 || startDaemon < 0 || startStore > startDaemon { + t.Errorf("State Store must start before archied; calls =\n%s", calls) + } + if stopUI < 0 || stopStore < 0 || stopUI > stopStore { + t.Errorf("UI must stop before the State Store; calls =\n%s", calls) + } + if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) { + t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err) + } + for _, name := range []string{"archied", "archie-state-store"} { + got, err := os.ReadFile(filepath.Join(binDir, name)) + if err != nil { + t.Fatal(err) + } + if string(got) != "old" { + t.Errorf("%s = %q after rollback, want the previous binary", name, got) + } + } +} diff --git a/scripts/archie-update-install b/scripts/archie-update-install index 2fb0bc620..da8ce6cfb 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -62,6 +62,17 @@ AGENT_IMAGE_BACKUP="$BIN_DIR/archie-agent-image.prev.tar" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-$(configured_task_db_path)}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" +# The archied release is a set of processes, not one binary. Listed in start +# order: the State Store owns archie.db and every other process dials it, and +# the Gateway serves the chat contract archied and the UI dial, so each comes +# up before its consumers. archie-playbooks is a CLI, so it is installed but +# never restarted. deployments/systemd-user-service.md is the reference +# deployment; installing archied alone leaves the rest on the previous +# release, which is why a 1.23.0 update failed its health check +# (archie-core-hbqk). +GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui" +GATEWAY_BINARIES="$GATEWAY_SERVICES archie-playbooks" + PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}" PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}" GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}" @@ -133,6 +144,25 @@ esac git -C "$source_dir" checkout --quiet --detach "$build_commit" cd "$source_dir" +# Refuse before touching anything if the release runs processes this host has +# no unit for. Installing the binaries anyway would leave archied dialling a +# State Store that nothing starts: it fails its health check and rolls back, +# which reads as a broken release rather than an incomplete deployment. +if [[ ",$components," == *,daemon,* ]]; then + missing_units="" + for svc in $GATEWAY_SERVICES; do + [ -d "$source_dir/cmd/$svc" ] || continue + if ! systemctl --user list-unit-files "$svc.service" --no-legend 2>/dev/null | grep -q .; then + missing_units="${missing_units:+$missing_units }$svc.service" + fi + done + if [ -n "$missing_units" ]; then + echo "refusing to install archied $GW: it runs as separate processes and this host has no unit for: $missing_units" >&2 + echo "create them from deployments/systemd-user-service.md, then retry the update" >&2 + exit 1 + fi +fi + # Refresh adapters from the approved release source. The running shell # continues with this copy; subsequent updates use the current contract. install -m755 "$source_dir/scripts/archie-update-install" "$BIN_DIR/archie-update-install" @@ -145,16 +175,34 @@ if [[ ",$components," == *,daemon,* ]] && [ -n "$TASK_DB_PATH" ] && [ -f "$TASK_ sqlite3 "$TASK_DB_PATH" ".backup '$TASK_DB_BACKUP'" fi +# built accumulates only the binaries this release actually ships, so an +# update to a release predating one of these processes stays buildable. +built="" if [[ ",$components," == *,daemon,* ]]; then - echo "==> building archied $GW" - go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" \ - -o "$tmp/archied" ./cmd/archied -fi -if [[ ",$components," == *,daemon,* ]]; then - echo "==> backing up current archied binary" - [ -f "$BIN_DIR/archied" ] && cp -f "$BIN_DIR/archied" "$BIN_DIR/archied.prev" - echo "==> installing archied" - install -m755 "$tmp/archied" "$BIN_DIR/archied" + ldflags="-X github.com/samcharles93/archie-core/internal/app/archied.gatewayVersion=$GW" + ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/app/archied.runtimeVersion=$RT" + ldflags="$ldflags -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" + for cmd in $GATEWAY_BINARIES; do + [ -d "./cmd/$cmd" ] || continue + echo "==> building $cmd $GW" + go build -ldflags "$ldflags" -o "$tmp/$cmd" "./cmd/$cmd" + built="${built:+$built }$cmd" + done + + echo "==> backing up current binaries" + for cmd in $built; do + if [ -f "$BIN_DIR/$cmd" ]; then + cp -f "$BIN_DIR/$cmd" "$BIN_DIR/$cmd.prev" + else + # New in this release: record that there was nothing here, so a + # rollback removes it instead of leaving a stray binary behind. + rm -f "$BIN_DIR/$cmd.prev" + fi + done + echo "==> installing $built" + for cmd in $built; do + install -m755 "$tmp/$cmd" "$BIN_DIR/$cmd" + done fi if [[ ",$components," == *,agent,* ]]; then echo "==> saving managed worker image $AGENT_IMAGE for rollback" @@ -172,6 +220,8 @@ systemd-run --user --unit=archied-update-restart --collect --quiet \ --setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \ --setenv=ARCHIE_UPDATE_THREAD_ID="${ARCHIE_UPDATE_THREAD_ID:-0}" \ --setenv=ARCHIE_UPDATE_COMPONENTS="$components" \ + --setenv=ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" \ + --setenv=ARCHIE_UPDATE_BINARIES="$built" \ --setenv=ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" \ --setenv=ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" \ --setenv=ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" \ diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index 07ebd886a..aa34fa9c3 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -22,6 +22,11 @@ REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" POLL_INTERVAL_SECONDS=3 COMPONENTS="${ARCHIE_UPDATE_COMPONENTS:-daemon,agent}" +# Set by archie-update-install: the units to restart, in start order, and the +# binaries it replaced. The defaults keep a hand-driven run on a pre- +# decomposition host working. +UNITS="${ARCHIE_UPDATE_UNITS:-archied}" +BINARIES="${ARCHIE_UPDATE_BINARIES:-archied}" json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' @@ -72,13 +77,58 @@ wait_healthy() { return 1 } -rollback_component() { - local name="$1" - if [ ! -f "$BIN_DIR/$name.prev" ]; then - echo "==> no backup for $name; cannot roll it back automatically" >&2 - return 1 +# manage_unit runs one systemctl action. archied is required: this whole +# mechanism exists to restart it, and a failed stop must abort the rollback +# rather than let it race a live process over the binary and the database. +# The sibling processes are best-effort, because archie-update-install has +# already refused the update if a unit for one of them was missing, and a +# probe that guesses wrong must not silently skip the required stop. +manage_unit() { + local action="$1" unit="$2" + if [ "$unit" = archied ]; then + systemctl --user "$action" "$unit.service" + return fi - cp -f "$BIN_DIR/$name.prev" "$BIN_DIR/$name" + systemctl --user "$action" "$unit.service" || true +} + +# start_units brings the deployment up in dependency order: the State Store +# before anything that dials it, the Gateway before archied and the UI. +start_units() { + local unit rc=0 + for unit in $UNITS; do + echo "==> restarting $unit.service" + manage_unit restart "$unit" || rc=1 + done + return $rc +} + +# stop_units tears down in reverse, so consumers are gone before the services +# they depend on. +stop_units() { + local unit ordered="" rc=0 + for unit in $UNITS; do + ordered="$unit${ordered:+ $ordered}" + done + for unit in $ordered; do + manage_unit stop "$unit" || rc=1 + done + return $rc +} + +# rollback_gateway restores every binary the install replaced. A binary with +# no .prev was introduced by the failed release, so it is removed rather than +# left behind pointing at processes the restored release does not run. +rollback_gateway() { + local cmd rc=0 + for cmd in $BINARIES; do + if [ -f "$BIN_DIR/$cmd.prev" ]; then + cp -f "$BIN_DIR/$cmd.prev" "$BIN_DIR/$cmd" || rc=1 + else + rm -f "$BIN_DIR/$cmd" || rc=1 + fi + done + return $rc } rollback_agent_image() { @@ -103,8 +153,7 @@ rollback_task_db() { rm -f "$TASK_DB_PATH-wal" "$TASK_DB_PATH-shm" } -echo "==> restarting archied.service" -systemctl --user restart archied.service +start_units if wait_healthy; then echo "==> new version is healthy" @@ -118,11 +167,11 @@ rolled_back=false # uses Restart=on-failure in the supported unit, so copying rollback state # while a crash-loop child is alive can race with SQLite and execute the new # binary after its files have been replaced. -if systemctl --user stop archied.service; then +if stop_units; then case "$COMPONENTS" in daemon) if [ -f "$BIN_DIR/archied.prev" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then - if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then + if { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then rolled_back=true fi else @@ -140,7 +189,7 @@ if systemctl --user stop archied.service; then # Preserve the old all-or-nothing rollback guarantee: never restore one # component and leave the other on the failed release. if [ -f "$BIN_DIR/archied.prev" ] && [ -f "$AGENT_IMAGE_BACKUP" ] && { [ -z "$TASK_DB_PATH" ] || [ -f "$TASK_DB_BACKUP" ]; }; then - if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_component archied; then + if rollback_agent_image && { [ -z "$TASK_DB_PATH" ] || rollback_task_db; } && rollback_gateway; then rolled_back=true fi else @@ -152,10 +201,10 @@ if systemctl --user stop archied.service; then ;; esac else - echo "==> could not stop archied.service; refusing rollback while it may be running" >&2 + echo "==> could not stop every unit; refusing rollback while one may be running" >&2 fi if [ "$rolled_back" = true ]; then - systemctl --user restart archied.service + start_units fi write_report "failed" "$rolled_back"