Files
archie/internal/releaseupdate/install_script_test.go
T
sam b8b477fb32 feat(statestore): offline backup, restore, validate and rollback
archie-state-store gains the offline recovery commands
docs/prds/runtime-control-plane.md ("Bootstrap, migration, and recovery")
asks for. They work on the task database file directly -- <db_path>-tasks.sqlite,
not db_path -- and need neither archied nor the Web UI, which matters because
the failure they exist for is the one that stops archied starting:

  backup   -db F -out F     consistent snapshot (VACUUM INTO), verified as a
                            usable store before it is published; safe against
                            a serving store
  restore  -db F -from F    replace the database with a snapshot, dropping the
                            replaced file's -wal/-shm
  validate -db F            readable, not corrupt, schema no newer than this
                            binary, every stored resource decoding under the
                            definition that owns it
  rollback -db F -kind K    replay an earlier revision of a stored resource
           [-revision N]    through the ordinary replace, recorded as a new
                            revision

No rollback RPC is added and none is needed: a revision carries its own value,
so restoring one is the same replace the dashboard and the channels already
use, by hand, with the store stopped.

"The State Store is stopped" is checked rather than assumed. The serving
process now takes an exclusive flock on <db>.lock for its whole life
(store.AcquireOwnership), and restore/rollback refuse with
`task database is owned by another process` while it holds it. backup is the
deliberate exception: the update installer runs inside the process it is
replacing and cannot stop the State Store.

scripts/archie-update-install takes its pre-flight snapshot through
`go run ./cmd/archie-state-store backup` from the release being installed
instead of its own sqlite3 shell, so the installer and an operator run one
implementation. The watchdog's refusal semantics are unchanged: an update that
cannot back up still must not start.

Docs: safe-change-and-recovery.md's "Recovery: not yet built" now describes
what exists.
2026-09-21 22:04:41 +10:00

866 lines
33 KiB
Go

package releaseupdate
import (
"bufio"
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
_ "modernc.org/sqlite"
)
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
assertCallContains(t, calls, "fetch --quiet --tags origin")
assertCallAbsent(t, calls, "git pull")
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallAbsent(t, calls, "docker compose build agent")
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
t.Fatalf("installed daemon = %q, want 1.13.0", got)
}
if _, found := result.Installed[ComponentAgent]; found {
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
}
if !result.RestartRequested {
t.Fatal("RestartRequested = false, want true for gateway update")
}
}
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallAbsent(t, calls, "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
t.Fatalf("installed agent = %q, want 1.10.0", got)
}
if _, found := result.Installed[ComponentDaemon]; found {
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
}
}
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallContains(t, calls, "go build", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
t.Fatalf("installed = %#v, want both release versions", result.Installed)
}
}
// Components are independently versioned (RELEASING.md), so the newest
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
// releases are the norm. Each changed component must be built from its own
// approved tag; forcing both tags onto one commit refuses every update once
// the two components' newest releases diverge.
func TestUpdateInstallBuildsEachComponentFromItsOwnTag(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
t.Fatalf("installed = %#v, want both release versions", result.Installed)
}
// The derived fake commits differ per tag, so these assertions fail if
// either component is built from the other's release.
daemonCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archied-v1.13.0")
agentCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archie-v1.10.0")
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
if daemonCheckout < 0 || agentCheckout < 0 || daemonBuild < 0 || agentImageBuild < 0 {
t.Fatalf("missing expected step: daemon checkout %d, daemon build %d, agent checkout %d, agent image build %d",
daemonCheckout, daemonBuild, agentCheckout, agentImageBuild)
}
if daemonCheckout > daemonBuild {
t.Errorf("daemon binaries must be built from the daemon tag's commit; calls = %#v", calls)
}
if agentCheckout > agentImageBuild {
t.Errorf("the managed worker image must be built from the agent tag's commit; calls = %#v", calls)
}
}
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
t.Helper()
for i, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return i
}
}
return -1
}
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
assertCallAbsent(t, calls, forbidden)
}
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
t.Fatalf("no-op result = %#v, want empty result without restart", result)
}
}
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
// the two: systemd-run starts the watchdog with a clean environment, so a
// health address the daemon derived reaches the probe only if the install
// script hands it over explicitly (archie-core-1r4g).
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
})
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
}
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
tests := []struct {
name string
components string
wantDaemon string
wantImageRollback bool
}{
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
if daemon != tt.wantDaemon {
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
}
if tt.wantImageRollback {
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
} else {
assertCallAbsent(t, calls, "docker image load")
}
if !report.RolledBack || report.HealthCheck != "failed" {
t.Fatalf("report = %#v, want failed rollback", report)
}
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
}
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
}
})
}
}
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
// no other test reaches: a daemon that comes back up. Every other watchdog
// test fakes curl as an immediate failure, so before this one the probe URL
// the script actually builds was never dialled and its default could point
// anywhere (archie-core-1r4g).
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
daemon := httptest.NewServer(mux)
defer daemon.Close()
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
env: map[string]string{
"ARCHIE_HEALTH_URL": daemon.URL,
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
},
})
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
if installed != "new daemon" {
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
}
assertCallAbsent(t, calls, "docker image load")
}
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
// script uses when nothing exports one. It has to be the address archied
// itself serves /healthz on ([health] listen); anything else times out and
// rolls back a healthy release.
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
report, _, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
})
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
}
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
createSQLiteDatabase(t, path, version)
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatal(err)
}
defer db.Close()
var version int
if err := db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil {
t.Fatal(err)
}
if version != 1 {
t.Fatalf("restored schema version = %d, want 1", version)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
stopIndex, copyIndex, startIndex := -1, -1, -1
for i, line := range callLines {
switch line {
case "systemctl --user stop archied.service":
stopIndex = i
case "systemctl --user restart archied.service":
startIndex = i
}
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
copyIndex = i
}
}
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
}
if startIndex < 0 || startIndex <= copyIndex {
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
}
}
func createSQLiteDatabase(t *testing.T, path string, version int) {
t.Helper()
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatalf("open database %s: %v", path, err)
}
defer db.Close()
if _, err := db.ExecContext(t.Context(), "PRAGMA user_version = "+strconv.Itoa(version)); err != nil {
t.Fatalf("set database version for %s: %v", path, err)
}
if _, err := db.ExecContext(t.Context(), "CREATE TABLE tasks (id INTEGER)"); err != nil {
t.Fatalf("create tasks table in %s: %v", path, err)
}
}
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
t.Helper()
result, calls, _ := runUpdateInstall(t, environment, false)
return result, calls
}
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
// returns the parsed result (empty when the run was expected to fail), the
// recorded calls, and the combined output.
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
t.Helper()
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
}
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
// file under the caller's control. Config-derived behaviour -- which database
// the updater backs up, which image it rebuilds -- is otherwise untestable,
// which is how the task-database backup path came to be wrong.
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
configPath := filepath.Join(work, "config.toml")
fakeDir := filepath.Join(work, "fake-bin")
callsPath := filepath.Join(work, "calls")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"archied", "archie-agent"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte("old "+name), 0o755); err != nil {
t.Fatal(err)
}
}
writeFakeCommand(t, fakeDir, "git", `
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
case "$*" in
*"clone "*)
source_dir="${@: -1}"
mkdir -p "$source_dir"
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do
mkdir -p "$source_dir/cmd/$c"
done
: > "$source_dir/docker-compose.yml"
;;
*"rev-parse --verify refs/tags/"*)
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
echo "${tag//\//-}"
;;
esac
`)
writeFakeCommand(t, fakeDir, "go", `
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
# The pre-flight task-database backup runs through the release's own
# archie-state-store subcommand, so a release whose recovery path cannot run
# must refuse the update.
if [ "$1" = "run" ]; then
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
exit 0
fi
out=""
while [ "$#" -gt 0 ]; do
if [ "$1" = "-o" ]; then out="$2"; break; fi
shift
done
mkdir -p "$(dirname "$out")"
printf 'built\n' > "$out"
`)
writeFakeCommand(t, fakeDir, "docker", `
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
if [ "$1" = image ] && [ "$2" = save ]; then
while [ "$#" -gt 0 ]; do
if [ "$1" = --output ]; then
: > "$2"
break
fi
shift
done
fi
`)
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "systemctl", `
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
if [ "$2" = list-unit-files ]; then
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
*" $3 "*) exit 0 ;;
esac
printf '%s enabled enabled\n' "$3"
fi
`)
writeFakeCommand(t, fakeDir, "install", `
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/install "$@"
`)
writeFakeCommand(t, fakeDir, "cp", `
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/cp "$@"
`)
writeFakeCommand(t, fakeDir, "sqlite3", `
printf '%s\n' "sqlite3 $*" >> "$ARCHIE_TEST_CALLS"
[ "${ARCHIE_TEST_SQLITE3_FAILS:-}" = "1" ] && exit 1
# Reproduce ".backup '<path>'" by writing a plausible backup beside the
# source, so the watchdog's existence check behaves as it would in production.
for arg in "$@"; do
case "$arg" in
.backup*) target="$(printf '%s' "$arg" | sed "s/^\.backup[[:space:]]*//; s/^'//; s/'$//")" ;;
esac
done
if [ -n "${target:-}" ]; then
mkdir -p "$(dirname "$target")"
printf 'backup\n' > "$target"
fi
`)
// The production adapter prepends /usr/local/go/bin. Run an otherwise
// byte-for-byte copy with the harness PATH left first so fake go records
// build intent without compiling or touching Docker/systemd.
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
if err != nil {
t.Fatal(err)
}
scriptPath := filepath.Join(work, "archie-update-install")
scriptData = []byte(strings.Replace(string(scriptData), `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1))
if err := os.WriteFile(scriptPath, scriptData, 0o755); err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, scriptPath)
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
"ARCHIE_TEST_SOURCE_DIR="+root,
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_CONFIG_PATH="+configPath,
"ARCHIE_TEST_CALLS="+callsPath,
)
for key, value := range environment {
cmd.Env = append(cmd.Env, key+"="+value)
}
output, err := cmd.CombinedOutput()
switch {
case wantErr && err == nil:
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
case !wantErr && err != nil:
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
if wantErr {
return Result{}, calls, string(output)
}
var result Result
scanner := bufio.NewScanner(strings.NewReader(string(output)))
found := false
for scanner.Scan() {
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
found = true
if err := json.Unmarshal([]byte(payload), &result); err != nil {
t.Fatalf("decode result %q: %v", payload, err)
}
}
}
if !found {
t.Fatalf("installer output has no result sentinel:\n%s", output)
}
return result, calls, string(output)
}
func writeFakeCommand(t *testing.T, dir, name, body string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
t.Fatal(err)
}
}
// watchdogRun configures one end-to-end run of the watchdog script. curl is
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
// which is what turns a health probe into a genuine test of the URL the
// watchdog builds rather than of a stub that ignores it.
type watchdogRun struct {
components string
curl string
env map[string]string
}
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
t.Helper()
return runUpdateWatchdog(t, watchdogRun{
components: components,
curl: `exit 1`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
})
}
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
fakeDir := filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
files := map[string]string{
"archied": "new daemon", "archied.prev": "old daemon",
"archie-agent-image.prev.tar": "saved old image",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
if run.curl != "" {
writeFakeCommand(t, fakeDir, "curl", run.curl)
}
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
callsPath := filepath.Join(work, "calls")
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
"ARCHIE_UPDATE_COMPONENTS="+run.components,
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
)
for key, value := range run.env {
cmd.Env = append(cmd.Env, key+"="+value)
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
reportData, err := os.ReadFile(reportPath)
if err != nil {
t.Fatal(err)
}
var report Report
if err := json.Unmarshal(reportData, &report); err != nil {
t.Fatal(err)
}
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
if err != nil {
t.Fatal(err)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
return report, string(daemon), calls
}
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
t.Helper()
for _, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return
}
}
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
}
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
t.Helper()
for _, call := range calls {
if strings.Contains(call, fragment) {
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
}
}
}
// An archied release is five processes. Installing the daemon binary alone
// left the State Store, Gateway and UI on the previous release, so archied
// came up against a State Store nothing had started, failed its health check
// and rolled back (archie-core-hbqk).
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
})
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} {
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
assertCallContains(t, calls, "install -m755", "/"+cmd)
}
assertCallAbsent(t, calls, "./cmd/archie-agent")
// The watchdog cannot restart what it is not told about.
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui")
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks")
}
// Installing binaries for processes the host has no unit for produces exactly
// the failure this bug was: archied dials a State Store nothing starts. Refuse
// before touching the host instead, and say what is missing.
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
_, calls, output := runUpdateInstall(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
}, true)
if !strings.Contains(output, "archie-state-store.service") {
t.Errorf("refusal must name the missing unit; output = %q", output)
}
// Nothing may be replaced on a host it would leave broken.
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
}
// The deployment is several processes, so the watchdog must cycle all of them,
// and in dependency order: the State Store owns archie.db and everything dials
// it, so it starts first and stops last (archie-core-hbqk).
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
// archie-ui is new in this release: it has no .prev, so rolling back must
// remove it rather than leave a binary from a release that was withdrawn.
for name, content := range map[string]string{
"archied": "new", "archied.prev": "old",
"archie-state-store": "new", "archie-state-store.prev": "old",
"archie-ui": "new",
} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
"ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
indexOf := func(want string) int {
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
if line == want {
return i
}
}
return -1
}
startStore := indexOf("systemctl --user restart archie-state-store.service")
startDaemon := indexOf("systemctl --user restart archied.service")
stopUI := indexOf("systemctl --user stop archie-ui.service")
stopStore := indexOf("systemctl --user stop archie-state-store.service")
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
t.Errorf("State Store must start before archied; calls =\n%s", calls)
}
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
}
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
}
for _, name := range []string{"archied", "archie-state-store"} {
got, err := os.ReadFile(filepath.Join(binDir, name))
if err != nil {
t.Fatal(err)
}
if string(got) != "old" {
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
}
}
}
// The updater must back up the database the daemon actually uses, which is
// `<db_path>-tasks.sqlite` (taskDBPath in internal/app/archied/main.go), not
// the configured path itself. The configured path may be a zero-byte
// placeholder while the real store sits beside it -- so backing the literal
// value up produces an empty backup, and a rollback then restores that empty
// file over the database and deletes its -wal/-shm. A rollback that destroys
// the task store is worse than no rollback.
//
// The snapshot is taken by the release's own recovery subcommand rather than
// by hand-rolled shell, so the installer and an operator run the same code and
// the two cannot drift apart.
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
realStore := configured + "-tasks.sqlite"
if err := os.WriteFile(configured, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(realStore, []byte("real task data"), 0o600); err != nil {
t.Fatal(err)
}
backup := filepath.Join(work, "backup.sqlite")
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
assertCallAbsent(t, calls, "sqlite3")
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
// The zero-byte placeholder is not the store, and must never be the thing
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+configured+" ")
}
// db_path also appears in other sections -- [indexing] owns its own database.
// Reading the first match in the file can select an unrelated database, so the
// value must come from the daemon's own top-level key.
func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
realStore := configured + "-tasks.sqlite"
indexStore := filepath.Join(work, "workspace-indexes.db")
for path, content := range map[string]string{realStore: "real task data", indexStore: "index data"} {
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
backup := filepath.Join(work, "backup.sqlite")
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
for _, call := range calls {
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
}
}
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+indexStore)
}
// A store that cannot be backed up must fail the update before anything is
// replaced. Installing with no usable rollback is how a failed update becomes
// a destroyed task store.
func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
if err := os.WriteFile(configured+"-tasks.sqlite", []byte("data"), 0o600); err != nil {
t.Fatal(err)
}
_, calls, output := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_GO_FAILS_RUN": "1",
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
if !strings.Contains(output, "task database") {
t.Errorf("refusal must name the task database; output = %q", output)
}
// Nothing of the release may be placed on the host, and no backup of the
// failure path may be attempted either.
for _, call := range calls {
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
t.Errorf("refused update still installed %q", call)
}
}
assertCallAbsent(t, calls, "arhied.prev")
}
// The sidecar is the only record of the managed agent version, and
// archie-update-check refuses anything but a bare release version, so a stray
// escape in the write leaves the installed agent reported as unknown forever.
func TestUpdateInstallWritesABareAgentVersionSidecar(t *testing.T) {
versionFile := filepath.Join(t.TempDir(), "archie-agent.version")
runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
"ARCHIE_AGENT_VERSION_FILE": versionFile,
})
recorded, err := os.ReadFile(versionFile)
if err != nil {
t.Fatal(err)
}
if string(recorded) != "1.10.0\n" {
t.Fatalf("sidecar = %q, want %q", recorded, "1.10.0\n")
}
}