A self-update to 1.23.0 failed its health check and rolled back. The adapter built and installed ./cmd/archied and nothing else, and the watchdog restarted archied.service and nothing else, so the new daemon came up against a State Store, Gateway and UI still on the previous release. Since 1.23.0 archied cannot boot without a reachable State Store (openStateStoreAdapter), which on a host that had never run one is an immediate exit, a failed health check, and a rollback that looks like a bad release rather than an incomplete deployment. Install every binary the reference deployment runs, versioned off the archied tag, and back each up so the rollback restores all of them together. A binary with no .prev was introduced by the failed release, so rollback removes it instead of leaving one behind that points at processes the restored release does not run. Restart in dependency order: the State Store owns archie.db and every other process dials it, the Gateway serves the contract archied and the UI dial, so each comes up before its consumers and stops after them. archied itself is cycled unconditionally and a failed stop still aborts the rollback, because a presence probe that guesses wrong must never skip the stop that keeps a live process off the binary and the database being restored. The siblings are best-effort, which is safe because the adapter now refuses the update outright, before touching the host, when a unit for one of them is missing, naming what to create. install.sh had the same single-binary assumption, so a fresh install produced a host that could not start either. Closes archie-core-hbqk.
665 lines
24 KiB
Go
665 lines
24 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
|
|
assertCallContains(t, calls, "fetch --quiet --tags origin")
|
|
assertCallAbsent(t, calls, "git pull")
|
|
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
|
|
assertCallContains(t, calls, "checkout --quiet --detach approved-release-commit")
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallAbsent(t, calls, "docker compose build agent")
|
|
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
|
|
t.Fatalf("installed daemon = %q, want 1.13.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentAgent]; found {
|
|
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
|
|
}
|
|
if !result.RestartRequested {
|
|
t.Fatal("RestartRequested = false, want true for gateway update")
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallAbsent(t, calls, "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
|
|
t.Fatalf("installed agent = %q, want 1.10.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentDaemon]; found {
|
|
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallContains(t, calls, "go build", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
|
|
assertCallAbsent(t, calls, forbidden)
|
|
}
|
|
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
|
|
t.Fatalf("no-op result = %#v, want empty result without restart", result)
|
|
}
|
|
}
|
|
|
|
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
|
|
// the two: systemd-run starts the watchdog with a clean environment, so a
|
|
// health address the daemon derived reaches the probe only if the install
|
|
// script hands it over explicitly (archie-core-1r4g).
|
|
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
|
|
})
|
|
|
|
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
|
|
}
|
|
|
|
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
components string
|
|
wantDaemon string
|
|
wantImageRollback bool
|
|
}{
|
|
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
|
|
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
|
|
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
|
|
if daemon != tt.wantDaemon {
|
|
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
|
|
}
|
|
if tt.wantImageRollback {
|
|
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
|
|
} else {
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
if !report.RolledBack || report.HealthCheck != "failed" {
|
|
t.Fatalf("report = %#v, want failed rollback", report)
|
|
}
|
|
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
|
|
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
|
|
}
|
|
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
|
|
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
|
|
// no other test reaches: a daemon that comes back up. Every other watchdog
|
|
// test fakes curl as an immediate failure, so before this one the probe URL
|
|
// the script actually builds was never dialled and its default could point
|
|
// anywhere (archie-core-1r4g).
|
|
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
|
daemon := httptest.NewServer(mux)
|
|
defer daemon.Close()
|
|
|
|
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
env: map[string]string{
|
|
"ARCHIE_HEALTH_URL": daemon.URL,
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
|
|
},
|
|
})
|
|
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
if installed != "new daemon" {
|
|
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
|
|
}
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
|
|
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
|
|
// script uses when nothing exports one. It has to be the address archied
|
|
// itself serves /healthz on ([health] listen); anything else times out and
|
|
// rolls back a healthy release.
|
|
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
|
|
report, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
|
|
})
|
|
|
|
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
}
|
|
|
|
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
|
|
createSQLiteDatabase(t, path, version)
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
|
|
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
db, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
var version int
|
|
if err := db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if version != 1 {
|
|
t.Fatalf("restored schema version = %d, want 1", version)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
|
stopIndex, copyIndex, startIndex := -1, -1, -1
|
|
for i, line := range callLines {
|
|
switch line {
|
|
case "systemctl --user stop archied.service":
|
|
stopIndex = i
|
|
case "systemctl --user restart archied.service":
|
|
startIndex = i
|
|
}
|
|
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
|
|
copyIndex = i
|
|
}
|
|
}
|
|
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
|
|
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
|
|
}
|
|
if startIndex < 0 || startIndex <= copyIndex {
|
|
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
|
|
}
|
|
}
|
|
|
|
func createSQLiteDatabase(t *testing.T, path string, version int) {
|
|
t.Helper()
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
t.Fatalf("open database %s: %v", path, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.ExecContext(t.Context(), "PRAGMA user_version = "+strconv.Itoa(version)); err != nil {
|
|
t.Fatalf("set database version for %s: %v", path, err)
|
|
}
|
|
if _, err := db.ExecContext(t.Context(), "CREATE TABLE tasks (id INTEGER)"); err != nil {
|
|
t.Fatalf("create tasks table in %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
|
t.Helper()
|
|
result, calls, _ := runUpdateInstall(t, environment, false)
|
|
return result, calls
|
|
}
|
|
|
|
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
|
|
// returns the parsed result (empty when the run was expected to fail), the
|
|
// recorded calls, and the combined output.
|
|
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
configPath := filepath.Join(work, "config.toml")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
callsPath := filepath.Join(work, "calls")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(configPath, []byte("[containers]\nimage = 'registry.example/archie-agent:stable'\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-agent"} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte("old "+name), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writeFakeCommand(t, fakeDir, "git", `
|
|
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"clone "*)
|
|
source_dir="${@: -1}"
|
|
mkdir -p "$source_dir"
|
|
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
|
|
for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do
|
|
mkdir -p "$source_dir/cmd/$c"
|
|
done
|
|
: > "$source_dir/docker-compose.yml"
|
|
;;
|
|
*"rev-parse --verify refs/tags/"*) echo approved-release-commit ;;
|
|
esac
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "go", `
|
|
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
|
out=""
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-o" ]; then out="$2"; break; fi
|
|
shift
|
|
done
|
|
mkdir -p "$(dirname "$out")"
|
|
printf 'built\n' > "$out"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "docker", `
|
|
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$1" = image ] && [ "$2" = save ]; then
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = --output ]; then
|
|
: > "$2"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "systemctl", `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$2" = list-unit-files ]; then
|
|
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
|
|
*" $3 "*) exit 0 ;;
|
|
esac
|
|
printf '%s enabled enabled\n' "$3"
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "install", `
|
|
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/install "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "cp", `
|
|
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/cp "$@"
|
|
`)
|
|
|
|
// The production adapter prepends /usr/local/go/bin. Run an otherwise
|
|
// byte-for-byte copy with the harness PATH left first so fake go records
|
|
// build intent without compiling or touching Docker/systemd.
|
|
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
scriptPath := filepath.Join(work, "archie-update-install")
|
|
scriptData = []byte(strings.Replace(string(scriptData), `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1))
|
|
if err := os.WriteFile(scriptPath, scriptData, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.CommandContext(ctx, scriptPath)
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
|
|
"ARCHIE_TEST_SOURCE_DIR="+root,
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_CONFIG_PATH="+configPath,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
)
|
|
for key, value := range environment {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
switch {
|
|
case wantErr && err == nil:
|
|
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
|
|
case !wantErr && err != nil:
|
|
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
if wantErr {
|
|
return Result{}, calls, string(output)
|
|
}
|
|
var result Result
|
|
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
|
found := false
|
|
for scanner.Scan() {
|
|
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
|
|
found = true
|
|
if err := json.Unmarshal([]byte(payload), &result); err != nil {
|
|
t.Fatalf("decode result %q: %v", payload, err)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("installer output has no result sentinel:\n%s", output)
|
|
}
|
|
return result, calls, string(output)
|
|
}
|
|
|
|
func writeFakeCommand(t *testing.T, dir, name, body string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(dir, name)
|
|
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// watchdogRun configures one end-to-end run of the watchdog script. curl is
|
|
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
|
|
// which is what turns a health probe into a genuine test of the URL the
|
|
// watchdog builds rather than of a stub that ignores it.
|
|
type watchdogRun struct {
|
|
components string
|
|
curl string
|
|
env map[string]string
|
|
}
|
|
|
|
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
|
|
t.Helper()
|
|
return runUpdateWatchdog(t, watchdogRun{
|
|
components: components,
|
|
curl: `exit 1`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
|
|
})
|
|
}
|
|
|
|
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := map[string]string{
|
|
"archied": "new daemon", "archied.prev": "old daemon",
|
|
"archie-agent-image.prev.tar": "saved old image",
|
|
}
|
|
for name, content := range files {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
|
|
if run.curl != "" {
|
|
writeFakeCommand(t, fakeDir, "curl", run.curl)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
callsPath := filepath.Join(work, "calls")
|
|
|
|
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
|
|
"ARCHIE_UPDATE_COMPONENTS="+run.components,
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
|
|
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
|
|
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
|
|
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
|
|
)
|
|
for key, value := range run.env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
reportData, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report Report
|
|
if err := json.Unmarshal(reportData, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
return report, string(daemon), calls
|
|
}
|
|
|
|
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
|
|
}
|
|
|
|
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
if strings.Contains(call, fragment) {
|
|
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An archied release is five processes. Installing the daemon binary alone
|
|
// left the State Store, Gateway and UI on the previous release, so archied
|
|
// came up against a State Store nothing had started, failed its health check
|
|
// and rolled back (archie-core-hbqk).
|
|
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
})
|
|
|
|
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} {
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
|
|
assertCallContains(t, calls, "install -m755", "/"+cmd)
|
|
}
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
// The watchdog cannot restart what it is not told about.
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui")
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks")
|
|
}
|
|
|
|
// Installing binaries for processes the host has no unit for produces exactly
|
|
// the failure this bug was: archied dials a State Store nothing starts. Refuse
|
|
// before touching the host instead, and say what is missing.
|
|
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
|
|
_, calls, output := runUpdateInstall(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-state-store.service") {
|
|
t.Errorf("refusal must name the missing unit; output = %q", output)
|
|
}
|
|
// Nothing may be replaced on a host it would leave broken.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// The deployment is several processes, so the watchdog must cycle all of them,
|
|
// and in dependency order: the State Store owns archie.db and everything dials
|
|
// it, so it starts first and stops last (archie-core-hbqk).
|
|
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// archie-ui is new in this release: it has no .prev, so rolling back must
|
|
// remove it rather than leave a binary from a release that was withdrawn.
|
|
for name, content := range map[string]string{
|
|
"archied": "new", "archied.prev": "old",
|
|
"archie-state-store": "new", "archie-state-store.prev": "old",
|
|
"archie-ui": "new",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
|
"ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
indexOf := func(want string) int {
|
|
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
|
if line == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
startStore := indexOf("systemctl --user restart archie-state-store.service")
|
|
startDaemon := indexOf("systemctl --user restart archied.service")
|
|
stopUI := indexOf("systemctl --user stop archie-ui.service")
|
|
stopStore := indexOf("systemctl --user stop archie-state-store.service")
|
|
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
|
|
t.Errorf("State Store must start before archied; calls =\n%s", calls)
|
|
}
|
|
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
|
|
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
|
|
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-state-store"} {
|
|
got, err := os.ReadFile(filepath.Join(binDir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(got) != "old" {
|
|
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
|
|
}
|
|
}
|
|
}
|