Files
archie/internal/releaseupdate/install_script_test.go
T
sam b12f05c970 fix(release): package and update archie-messaging on a host install
The v1.30.0 extraction moved Telegram, email and webhook out of archied and into
cmd/archie-messaging, and nothing packaged it. dist-zip built and copied only the
five older binaries, and scripts/archie-update-install omitted it from both its
service and binary lists. A host upgraded across v1.30.0 therefore ran on with no
messaging process at all: archied logged that it stopped serving the telegram
gateway at the extraction, nothing replaced it, and the bot was dead for two days
with no error anywhere -- the daemon looked healthy throughout (archie-core-1c01).

Fixed in all four places the host component set was enumerated: the zip's build
loop, the zip's copy list, the installer's GATEWAY_SERVICES (messaging is a
service, so it is installed AND restarted) and its GATEWAY_BINARIES. The comment
claiming every process in the documented deployment ships in the zip is now true,
and says what makes it true.

The duplication was the actual defect, so the sets are pinned to cmd/ instead:
TestHostCommandListsCannotDrift derives the host-run commands from cmd/ and
asserts all four lists, plus the reference runbook's units, agree with it. Adding
a command or extracting one fails the test until it is packaged -- the check that
would have caught this on the day. The installer's own test grew the same
coverage, and the reason it passed while the bug was live is worth recording: its
fixture created only the stale cmd/ set, so the script's `[ -d ./cmd/$cmd ]` guard
silently skipped messaging.

Documentation, because the installer now refuses an update when a service unit is
missing and sends the operator to the runbook -- so a service the runbook omits is
a dead end: a messaging unit in deployments/systemd-user-service.md, the component
list and start sequence in deployments/INSTRUCTIONS.md (which ships inside the
zip), and the packaged-binary list in RELEASING.md.

Verified: task check exit 0, including test:tools and test:ui. Both new guards
were made to fail on purpose: reintroducing each omission produces "missing
archie-messaging" and "no call with fragments [./cmd/archie-messaging]".
2026-09-22 05:42:17 +10:00

866 lines
33 KiB
Go

package releaseupdate
import (
"bufio"
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
_ "modernc.org/sqlite"
)
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
assertCallContains(t, calls, "fetch --quiet --tags origin")
assertCallAbsent(t, calls, "git pull")
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallAbsent(t, calls, "docker compose build agent")
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
t.Fatalf("installed daemon = %q, want 1.13.0", got)
}
if _, found := result.Installed[ComponentAgent]; found {
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
}
if !result.RestartRequested {
t.Fatal("RestartRequested = false, want true for gateway update")
}
}
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallAbsent(t, calls, "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
t.Fatalf("installed agent = %q, want 1.10.0", got)
}
if _, found := result.Installed[ComponentDaemon]; found {
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
}
}
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallContains(t, calls, "go build", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
t.Fatalf("installed = %#v, want both release versions", result.Installed)
}
}
// Components are independently versioned (RELEASING.md), so the newest
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
// releases are the norm. Each changed component must be built from its own
// approved tag; forcing both tags onto one commit refuses every update once
// the two components' newest releases diverge.
func TestUpdateInstallBuildsEachComponentFromItsOwnTag(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
t.Fatalf("installed = %#v, want both release versions", result.Installed)
}
// The derived fake commits differ per tag, so these assertions fail if
// either component is built from the other's release.
daemonCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archied-v1.13.0")
agentCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archie-v1.10.0")
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
if daemonCheckout < 0 || agentCheckout < 0 || daemonBuild < 0 || agentImageBuild < 0 {
t.Fatalf("missing expected step: daemon checkout %d, daemon build %d, agent checkout %d, agent image build %d",
daemonCheckout, daemonBuild, agentCheckout, agentImageBuild)
}
if daemonCheckout > daemonBuild {
t.Errorf("daemon binaries must be built from the daemon tag's commit; calls = %#v", calls)
}
if agentCheckout > agentImageBuild {
t.Errorf("the managed worker image must be built from the agent tag's commit; calls = %#v", calls)
}
}
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
t.Helper()
for i, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return i
}
}
return -1
}
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
assertCallAbsent(t, calls, forbidden)
}
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
t.Fatalf("no-op result = %#v, want empty result without restart", result)
}
}
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
// the two: systemd-run starts the watchdog with a clean environment, so a
// health address the daemon derived reaches the probe only if the install
// script hands it over explicitly (archie-core-1r4g).
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
})
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
}
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
tests := []struct {
name string
components string
wantDaemon string
wantImageRollback bool
}{
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
if daemon != tt.wantDaemon {
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
}
if tt.wantImageRollback {
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
} else {
assertCallAbsent(t, calls, "docker image load")
}
if !report.RolledBack || report.HealthCheck != "failed" {
t.Fatalf("report = %#v, want failed rollback", report)
}
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
}
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
}
})
}
}
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
// no other test reaches: a daemon that comes back up. Every other watchdog
// test fakes curl as an immediate failure, so before this one the probe URL
// the script actually builds was never dialled and its default could point
// anywhere (archie-core-1r4g).
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
daemon := httptest.NewServer(mux)
defer daemon.Close()
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
env: map[string]string{
"ARCHIE_HEALTH_URL": daemon.URL,
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
},
})
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
if installed != "new daemon" {
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
}
assertCallAbsent(t, calls, "docker image load")
}
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
// script uses when nothing exports one. It has to be the address archied
// itself serves /healthz on ([health] listen); anything else times out and
// rolls back a healthy release.
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
report, _, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
})
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
}
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
createSQLiteDatabase(t, path, version)
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
db, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatal(err)
}
defer db.Close()
var version int
if err := db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil {
t.Fatal(err)
}
if version != 1 {
t.Fatalf("restored schema version = %d, want 1", version)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
stopIndex, copyIndex, startIndex := -1, -1, -1
for i, line := range callLines {
switch line {
case "systemctl --user stop archied.service":
stopIndex = i
case "systemctl --user restart archied.service":
startIndex = i
}
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
copyIndex = i
}
}
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
}
if startIndex < 0 || startIndex <= copyIndex {
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
}
}
func createSQLiteDatabase(t *testing.T, path string, version int) {
t.Helper()
db, err := sql.Open("sqlite", path)
if err != nil {
t.Fatalf("open database %s: %v", path, err)
}
defer db.Close()
if _, err := db.ExecContext(t.Context(), "PRAGMA user_version = "+strconv.Itoa(version)); err != nil {
t.Fatalf("set database version for %s: %v", path, err)
}
if _, err := db.ExecContext(t.Context(), "CREATE TABLE tasks (id INTEGER)"); err != nil {
t.Fatalf("create tasks table in %s: %v", path, err)
}
}
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
t.Helper()
result, calls, _ := runUpdateInstall(t, environment, false)
return result, calls
}
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
// returns the parsed result (empty when the run was expected to fail), the
// recorded calls, and the combined output.
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
t.Helper()
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
}
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
// file under the caller's control. Config-derived behaviour -- which database
// the updater backs up, which image it rebuilds -- is otherwise untestable,
// which is how the task-database backup path came to be wrong.
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
configPath := filepath.Join(work, "config.toml")
fakeDir := filepath.Join(work, "fake-bin")
callsPath := filepath.Join(work, "calls")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"archied", "archie-agent"} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte("old "+name), 0o755); err != nil {
t.Fatal(err)
}
}
writeFakeCommand(t, fakeDir, "git", `
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
case "$*" in
*"clone "*)
source_dir="${@: -1}"
mkdir -p "$source_dir"
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
for c in archied archie-gateway archie-state-store archie-ui archie-messaging archie-playbooks archie-agent; do
mkdir -p "$source_dir/cmd/$c"
done
: > "$source_dir/docker-compose.yml"
;;
*"rev-parse --verify refs/tags/"*)
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
echo "${tag//\//-}"
;;
esac
`)
writeFakeCommand(t, fakeDir, "go", `
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
# The pre-flight task-database backup runs through the release's own
# archie-state-store subcommand, so a release whose recovery path cannot run
# must refuse the update.
if [ "$1" = "run" ]; then
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
exit 0
fi
out=""
while [ "$#" -gt 0 ]; do
if [ "$1" = "-o" ]; then out="$2"; break; fi
shift
done
mkdir -p "$(dirname "$out")"
printf 'built\n' > "$out"
`)
writeFakeCommand(t, fakeDir, "docker", `
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
if [ "$1" = image ] && [ "$2" = save ]; then
while [ "$#" -gt 0 ]; do
if [ "$1" = --output ]; then
: > "$2"
break
fi
shift
done
fi
`)
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "systemctl", `
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
if [ "$2" = list-unit-files ]; then
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
*" $3 "*) exit 0 ;;
esac
printf '%s enabled enabled\n' "$3"
fi
`)
writeFakeCommand(t, fakeDir, "install", `
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/install "$@"
`)
writeFakeCommand(t, fakeDir, "cp", `
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/cp "$@"
`)
writeFakeCommand(t, fakeDir, "sqlite3", `
printf '%s\n' "sqlite3 $*" >> "$ARCHIE_TEST_CALLS"
[ "${ARCHIE_TEST_SQLITE3_FAILS:-}" = "1" ] && exit 1
# Reproduce ".backup '<path>'" by writing a plausible backup beside the
# source, so the watchdog's existence check behaves as it would in production.
for arg in "$@"; do
case "$arg" in
.backup*) target="$(printf '%s' "$arg" | sed "s/^\.backup[[:space:]]*//; s/^'//; s/'$//")" ;;
esac
done
if [ -n "${target:-}" ]; then
mkdir -p "$(dirname "$target")"
printf 'backup\n' > "$target"
fi
`)
// The production adapter prepends /usr/local/go/bin. Run an otherwise
// byte-for-byte copy with the harness PATH left first so fake go records
// build intent without compiling or touching Docker/systemd.
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
if err != nil {
t.Fatal(err)
}
scriptPath := filepath.Join(work, "archie-update-install")
scriptData = []byte(strings.Replace(string(scriptData), `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1))
if err := os.WriteFile(scriptPath, scriptData, 0o755); err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, scriptPath)
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
"ARCHIE_TEST_SOURCE_DIR="+root,
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_CONFIG_PATH="+configPath,
"ARCHIE_TEST_CALLS="+callsPath,
)
for key, value := range environment {
cmd.Env = append(cmd.Env, key+"="+value)
}
output, err := cmd.CombinedOutput()
switch {
case wantErr && err == nil:
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
case !wantErr && err != nil:
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
if wantErr {
return Result{}, calls, string(output)
}
var result Result
scanner := bufio.NewScanner(strings.NewReader(string(output)))
found := false
for scanner.Scan() {
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
found = true
if err := json.Unmarshal([]byte(payload), &result); err != nil {
t.Fatalf("decode result %q: %v", payload, err)
}
}
}
if !found {
t.Fatalf("installer output has no result sentinel:\n%s", output)
}
return result, calls, string(output)
}
func writeFakeCommand(t *testing.T, dir, name, body string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
t.Fatal(err)
}
}
// watchdogRun configures one end-to-end run of the watchdog script. curl is
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
// which is what turns a health probe into a genuine test of the URL the
// watchdog builds rather than of a stub that ignores it.
type watchdogRun struct {
components string
curl string
env map[string]string
}
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
t.Helper()
return runUpdateWatchdog(t, watchdogRun{
components: components,
curl: `exit 1`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
})
}
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
fakeDir := filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
files := map[string]string{
"archied": "new daemon", "archied.prev": "old daemon",
"archie-agent-image.prev.tar": "saved old image",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
if run.curl != "" {
writeFakeCommand(t, fakeDir, "curl", run.curl)
}
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
callsPath := filepath.Join(work, "calls")
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
"ARCHIE_UPDATE_COMPONENTS="+run.components,
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
)
for key, value := range run.env {
cmd.Env = append(cmd.Env, key+"="+value)
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
reportData, err := os.ReadFile(reportPath)
if err != nil {
t.Fatal(err)
}
var report Report
if err := json.Unmarshal(reportData, &report); err != nil {
t.Fatal(err)
}
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
if err != nil {
t.Fatal(err)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
return report, string(daemon), calls
}
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
t.Helper()
for _, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return
}
}
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
}
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
t.Helper()
for _, call := range calls {
if strings.Contains(call, fragment) {
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
}
}
}
// An archied release is five processes. Installing the daemon binary alone
// left the State Store, Gateway and UI on the previous release, so archied
// came up against a State Store nothing had started, failed its health check
// and rolled back (archie-core-hbqk).
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
})
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-messaging", "archie-playbooks"} {
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
assertCallContains(t, calls, "install -m755", "/"+cmd)
}
assertCallAbsent(t, calls, "./cmd/archie-agent")
// The watchdog cannot restart what it is not told about.
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui archie-messaging")
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-messaging archie-playbooks")
}
// Installing binaries for processes the host has no unit for produces exactly
// the failure this bug was: archied dials a State Store nothing starts. Refuse
// before touching the host instead, and say what is missing.
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
_, calls, output := runUpdateInstall(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
}, true)
if !strings.Contains(output, "archie-state-store.service") {
t.Errorf("refusal must name the missing unit; output = %q", output)
}
// Nothing may be replaced on a host it would leave broken.
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
}
// The deployment is several processes, so the watchdog must cycle all of them,
// and in dependency order: the State Store owns archie.db and everything dials
// it, so it starts first and stops last (archie-core-hbqk).
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
// archie-ui is new in this release: it has no .prev, so rolling back must
// remove it rather than leave a binary from a release that was withdrawn.
for name, content := range map[string]string{
"archied": "new", "archied.prev": "old",
"archie-state-store": "new", "archie-state-store.prev": "old",
"archie-ui": "new",
} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
"ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
indexOf := func(want string) int {
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
if line == want {
return i
}
}
return -1
}
startStore := indexOf("systemctl --user restart archie-state-store.service")
startDaemon := indexOf("systemctl --user restart archied.service")
stopUI := indexOf("systemctl --user stop archie-ui.service")
stopStore := indexOf("systemctl --user stop archie-state-store.service")
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
t.Errorf("State Store must start before archied; calls =\n%s", calls)
}
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
}
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
}
for _, name := range []string{"archied", "archie-state-store"} {
got, err := os.ReadFile(filepath.Join(binDir, name))
if err != nil {
t.Fatal(err)
}
if string(got) != "old" {
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
}
}
}
// The updater must back up the database the daemon actually uses, which is
// `<db_path>-tasks.sqlite` (taskDBPath in internal/app/archied/main.go), not
// the configured path itself. The configured path may be a zero-byte
// placeholder while the real store sits beside it -- so backing the literal
// value up produces an empty backup, and a rollback then restores that empty
// file over the database and deletes its -wal/-shm. A rollback that destroys
// the task store is worse than no rollback.
//
// The snapshot is taken by the release's own recovery subcommand rather than
// by hand-rolled shell, so the installer and an operator run the same code and
// the two cannot drift apart.
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
realStore := configured + "-tasks.sqlite"
if err := os.WriteFile(configured, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(realStore, []byte("real task data"), 0o600); err != nil {
t.Fatal(err)
}
backup := filepath.Join(work, "backup.sqlite")
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
assertCallAbsent(t, calls, "sqlite3")
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
// The zero-byte placeholder is not the store, and must never be the thing
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+configured+" ")
}
// db_path also appears in other sections -- [indexing] owns its own database.
// Reading the first match in the file can select an unrelated database, so the
// value must come from the daemon's own top-level key.
func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
realStore := configured + "-tasks.sqlite"
indexStore := filepath.Join(work, "workspace-indexes.db")
for path, content := range map[string]string{realStore: "real task data", indexStore: "index data"} {
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
backup := filepath.Join(work, "backup.sqlite")
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TASK_DB_BACKUP": backup,
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
for _, call := range calls {
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
}
}
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+indexStore)
}
// A store that cannot be backed up must fail the update before anything is
// replaced. Installing with no usable rollback is how a failed update becomes
// a destroyed task store.
func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
work := t.TempDir()
configured := filepath.Join(work, "archie.db")
if err := os.WriteFile(configured+"-tasks.sqlite", []byte("data"), 0o600); err != nil {
t.Fatal(err)
}
_, calls, output := runUpdateInstallWithConfig(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_GO_FAILS_RUN": "1",
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
if !strings.Contains(output, "task database") {
t.Errorf("refusal must name the task database; output = %q", output)
}
// Nothing of the release may be placed on the host, and no backup of the
// failure path may be attempted either.
for _, call := range calls {
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
t.Errorf("refused update still installed %q", call)
}
}
assertCallAbsent(t, calls, "arhied.prev")
}
// The sidecar is the only record of the managed agent version, and
// archie-update-check refuses anything but a bare release version, so a stray
// escape in the write leaves the installed agent reported as unknown forever.
func TestUpdateInstallWritesABareAgentVersionSidecar(t *testing.T) {
versionFile := filepath.Join(t.TempDir(), "archie-agent.version")
runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
"ARCHIE_AGENT_VERSION_FILE": versionFile,
})
recorded, err := os.ReadFile(versionFile)
if err != nil {
t.Fatal(err)
}
if string(recorded) != "1.10.0\n" {
t.Fatalf("sidecar = %q, want %q", recorded, "1.10.0\n")
}
}