Files
archie/internal/releaseupdate/install_script_test.go
T
sam aa24551d98 fix(update): treat the runtime as the release version; stamp it in dist-zip
The update check still read the retired archie/v* tag stream, so a host on
1.41.1 was offered runtime 1.35.0, and the installer built the agent image
from that stale tag. Both now use the release's archied/v* tag. The dist-zip
build also stamps runtimeVersion, so /version stops reporting 'dev'.
2026-09-25 01:58:20 +10:00

860 lines
33 KiB
Go

package releaseupdate
import (
"bufio"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
)
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
assertCallContains(t, calls, "fetch --quiet --tags origin")
assertCallAbsent(t, calls, "git pull")
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallAbsent(t, calls, "docker compose build agent")
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
t.Fatalf("installed daemon = %q, want 1.13.0", got)
}
if _, found := result.Installed[ComponentAgent]; found {
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
}
if !result.RestartRequested {
t.Fatal("RestartRequested = false, want true for gateway update")
}
}
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallAbsent(t, calls, "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
t.Fatalf("installed agent = %q, want 1.10.0", got)
}
if _, found := result.Installed[ComponentDaemon]; found {
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
}
}
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
})
assertCallContains(t, calls, "go build", "./cmd/archied")
assertCallAbsent(t, calls, "./cmd/archie-agent")
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
t.Fatalf("installed = %#v, want both release versions", result.Installed)
}
}
// Components are independently versioned (RELEASING.md), so the newest
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
// releases are the norm. Each changed component must be built from its own
// approved tag; forcing both tags onto one commit refuses every update once
// the two components' newest releases diverge.
// One version covers every component, so the managed worker image is built
// from the release's own archied/v* tag -- never from a stale archie/v* tag.
func TestUpdateInstallBuildsTheAgentImageFromTheReleaseTag(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_AGENT_VERSION": "1.13.0",
})
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.13.0" {
t.Fatalf("installed = %#v, want the release version for both", result.Installed)
}
assertCallAbsent(t, calls, "archie-v1.13.0")
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
lastCheckout := -1
for i, call := range calls {
if strings.Contains(call, "checkout --quiet --detach") && i < agentImageBuild {
lastCheckout = i
}
}
if daemonBuild < 0 || agentImageBuild < 0 || lastCheckout < 0 {
t.Fatalf("missing expected step: daemon build %d, agent image build %d, checkout %d", daemonBuild, agentImageBuild, lastCheckout)
}
if !strings.Contains(calls[lastCheckout], "archied-v1.13.0") {
t.Errorf("image built after checking out %q, want the archied/v1.13.0 release; calls = %#v", calls[lastCheckout], calls)
}
}
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
t.Helper()
for i, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return i
}
}
return -1
}
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
result, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
})
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
assertCallAbsent(t, calls, forbidden)
}
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
t.Fatalf("no-op result = %#v, want empty result without restart", result)
}
}
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
// the two: systemd-run starts the watchdog with a clean environment, so a
// health address the daemon derived reaches the probe only if the install
// script hands it over explicitly (archie-core-1r4g).
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
})
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
}
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
tests := []struct {
name string
components string
wantDaemon string
wantImageRollback bool
}{
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
if daemon != tt.wantDaemon {
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
}
if tt.wantImageRollback {
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
} else {
assertCallAbsent(t, calls, "docker image load")
}
if !report.RolledBack || report.HealthCheck != "failed" {
t.Fatalf("report = %#v, want failed rollback", report)
}
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
}
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
}
})
}
}
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
// no other test reaches: a daemon that comes back up. Every other watchdog
// test fakes curl as an immediate failure, so before this one the probe URL
// the script actually builds was never dialled and its default could point
// anywhere (archie-core-1r4g).
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
daemon := httptest.NewServer(mux)
defer daemon.Close()
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
env: map[string]string{
"ARCHIE_HEALTH_URL": daemon.URL,
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
},
})
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
if installed != "new daemon" {
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
}
assertCallAbsent(t, calls, "docker image load")
}
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
// script uses when nothing exports one. It has to be the address archied
// itself serves /healthz on ([health] listen); anything else times out and
// rolls back a healthy release.
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
report, _, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
})
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
}
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
t.Helper()
result, calls, _ := runUpdateInstall(t, environment, false)
return result, calls
}
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
// returns the parsed result (empty when the run was expected to fail), the
// recorded calls, and the combined output.
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
t.Helper()
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
}
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
// file under the caller's control. Config-derived behaviour -- which database
// the updater backs up, which image it rebuilds -- is otherwise untestable,
// which is how the task-database backup path came to be wrong.
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
t.Helper()
return runUpdateInstallEdited(t, environment, configBody, nil, wantErr)
}
// runUpdateInstallEdited is runUpdateInstallWithConfig with the ability to edit
// the installer's text before it runs. The service list is hardcoded, so the
// only way to test the coverage guard against the stale list it exists for is to
// change that text -- which is exactly the state the guard must catch.
func runUpdateInstallEdited(t *testing.T, environment map[string]string, configBody string, edit func(string) string, wantErr bool) (Result, []string, string) {
t.Helper()
return runUpdateInstallFull(t, environment, configBody, nil, edit, wantErr)
}
// runUpdateInstallFull is runUpdateInstallEdited with control over the versions
// already installed in the bin directory, so a skewed host can be built on
// purpose (archie-core-k94o).
func runUpdateInstallFull(t *testing.T, environment map[string]string, configBody string, installed map[string]string, edit func(string) string, wantErr bool) (Result, []string, string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
configPath := filepath.Join(work, "config.toml")
fakeDir := filepath.Join(work, "fake-bin")
callsPath := filepath.Join(work, "calls")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
t.Fatal(err)
}
previousVersion := environment["ARCHIE_UPDATE_DAEMON_PREVIOUS"]
if previousVersion == "" {
previousVersion = "dev"
}
writeFakeBinary(t, binDir, "archied", previousVersion)
if err := os.WriteFile(filepath.Join(binDir, "archie-agent"), []byte("old archie-agent"), 0o755); err != nil {
t.Fatal(err)
}
for name, version := range installed {
writeFakeBinary(t, binDir, name, version)
}
writeFakeCommand(t, fakeDir, "git", `
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
case "$*" in
*"clone "*)
source_dir="${@: -1}"
mkdir -p "$source_dir"
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
for c in archied archie-gateway archie-state-store archie-ui archie-messaging archie-playbooks archie-agent; do
mkdir -p "$source_dir/cmd/$c"
done
: > "$source_dir/docker-compose.yml"
;;
*"rev-parse --verify refs/tags/"*)
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
echo "${tag//\//-}"
;;
esac
`)
writeFakeCommand(t, fakeDir, "go", `
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
# The pre-flight database snapshot runs through the release's own
# archie-state-store subcommand, so a release whose recovery path cannot run
# must refuse the update.
if [ "$1" = "run" ]; then
[ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && exit 1
exit 0
fi
out=""
name=""
version="dev"
while [ "$#" -gt 0 ]; do
if [ "$1" = "-o" ]; then out="$2"; fi
case "$1" in
./cmd/*) name="${1#./cmd/}" ;;
*buildinfo.Version=*)
version="${1##*buildinfo.Version=}"
version="${version%% *}"
;;
esac
shift
done
mkdir -p "$(dirname "$out")"
# The installed binary must answer -version the way a real one does, or the
# updater's skew check cannot read it.
printf '#!/usr/bin/env bash\necho "%s %s"\n' "$name" "$version" > "$out"
chmod 755 "$out"
`)
writeFakeCommand(t, fakeDir, "docker", `
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
if [ "$1" = image ] && [ "$2" = save ]; then
while [ "$#" -gt 0 ]; do
if [ "$1" = --output ]; then
: > "$2"
break
fi
shift
done
fi
`)
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "systemctl", `
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
if [ "$2" = list-unit-files ]; then
# The reference deployment runs exactly these five as units. A command with
# no unit -- archie-playbooks, archie-agent -- has to look unitless, or the
# coverage guard would mistake every cmd/ directory for a host service.
present="${ARCHIE_TEST_PRESENT_UNITS:-archied.service archie-gateway.service archie-state-store.service archie-ui.service archie-messaging.service}"
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
*" $3 "*) exit 0 ;;
esac
case " $present " in
*" $3 "*) printf '%s enabled enabled\n' "$3" ;;
esac
fi
`)
writeFakeCommand(t, fakeDir, "install", `
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/install "$@"
`)
writeFakeCommand(t, fakeDir, "cp", `
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
/usr/bin/cp "$@"
`)
// The production adapter prepends /usr/local/go/bin. Run an otherwise
// byte-for-byte copy with the harness PATH left first so fake go records
// build intent without compiling or touching Docker/systemd.
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
if err != nil {
t.Fatal(err)
}
script := string(scriptData)
if edit != nil {
script = edit(script)
}
scriptPath := filepath.Join(work, "archie-update-install")
script = strings.Replace(script, `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1)
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
cmd := exec.CommandContext(ctx, scriptPath)
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
"ARCHIE_TEST_SOURCE_DIR="+root,
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_CONFIG_PATH="+configPath,
"ARCHIE_TEST_CALLS="+callsPath,
)
for key, value := range environment {
cmd.Env = append(cmd.Env, key+"="+value)
}
output, err := cmd.CombinedOutput()
switch {
case wantErr && err == nil:
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
case !wantErr && err != nil:
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
if wantErr {
return Result{}, calls, string(output)
}
var result Result
scanner := bufio.NewScanner(strings.NewReader(string(output)))
found := false
for scanner.Scan() {
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
found = true
if err := json.Unmarshal([]byte(payload), &result); err != nil {
t.Fatalf("decode result %q: %v", payload, err)
}
}
}
if !found {
t.Fatalf("installer output has no result sentinel:\n%s", output)
}
return result, calls, string(output)
}
func writeFakeBinary(t *testing.T, dir, name, version string) {
t.Helper()
body := "#!/usr/bin/env bash\necho " + strconv.Quote(name+" "+version) + "\n"
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
t.Fatal(err)
}
}
func writeFakeCommand(t *testing.T, dir, name, body string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, name)
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
t.Fatal(err)
}
}
// watchdogRun configures one end-to-end run of the watchdog script. curl is
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
// which is what turns a health probe into a genuine test of the URL the
// watchdog builds rather than of a stub that ignores it.
type watchdogRun struct {
components string
curl string
// systemctl replaces the fake systemctl on PATH, so a case can observe or
// condition what the watchdog does between units. Empty keeps the inert
// `exit 0` stub.
systemctl string
env map[string]string
}
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
t.Helper()
return runUpdateWatchdog(t, watchdogRun{
components: components,
curl: `exit 1`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
})
}
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
t.Helper()
ctx := t.Context()
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir := filepath.Join(work, "bin")
fakeDir := filepath.Join(work, "fake-bin")
reportPath := filepath.Join(work, "report.json")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
files := map[string]string{
"archied": "new daemon", "archied.prev": "old daemon",
"archie-agent-image.prev.tar": "saved old image",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
systemctl := run.systemctl
if systemctl == "" {
systemctl = `exit 0`
}
writeFakeCommand(t, fakeDir, "systemctl", systemctl)
if run.curl != "" {
writeFakeCommand(t, fakeDir, "curl", run.curl)
}
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
callsPath := filepath.Join(work, "calls")
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(
os.Environ(),
"PATH="+fakeDir+":"+os.Getenv("PATH"),
"ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
"ARCHIE_UPDATE_COMPONENTS="+run.components,
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
)
for key, value := range run.env {
cmd.Env = append(cmd.Env, key+"="+value)
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
reportData, err := os.ReadFile(reportPath)
if err != nil {
t.Fatal(err)
}
var report Report
if err := json.Unmarshal(reportData, &report); err != nil {
t.Fatal(err)
}
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
if err != nil {
t.Fatal(err)
}
callsData, err := os.ReadFile(callsPath)
if err != nil && !os.IsNotExist(err) {
t.Fatal(err)
}
var calls []string
if len(callsData) > 0 {
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
}
return report, string(daemon), calls
}
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
t.Helper()
for _, call := range calls {
matched := true
for _, fragment := range fragments {
matched = matched && strings.Contains(call, fragment)
}
if matched {
return
}
}
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
}
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
t.Helper()
for _, call := range calls {
if strings.Contains(call, fragment) {
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
}
}
}
// An archied release is five processes. Installing the daemon binary alone
// left the State Store, Gateway and UI on the previous release, so archied
// came up against a State Store nothing had started, failed its health check
// and rolled back (archie-core-hbqk).
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
})
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-messaging", "archie-playbooks"} {
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
assertCallContains(t, calls, "install -m755", "/"+cmd)
}
assertCallAbsent(t, calls, "./cmd/archie-agent")
// The watchdog cannot restart what it is not told about.
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui archie-messaging")
assertCallContains(t, calls, "systemd-run",
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-messaging archie-playbooks")
}
// Installing binaries for processes the host has no unit for produces exactly
// the failure this bug was: archied dials a State Store nothing starts. Refuse
// before touching the host instead, and say what is missing.
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
_, calls, output := runUpdateInstall(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
}, true)
if !strings.Contains(output, "archie-state-store.service") {
t.Errorf("refusal must name the missing unit; output = %q", output)
}
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
// Nothing may be replaced on a host it would leave broken.
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
}
// The other half of the same contract, and the failure that actually happened
// (archie-core-1faq): the installed updater's GATEWAY_SERVICES predated
// archie-messaging, so the update neither backed it up nor restarted it and a
// hand-installed binary kept serving under an otherwise updated host. A unit
// this host runs must be named by the list, or the update refuses and says which.
func TestUpdateInstallRefusesWhenTheServiceListOmitsAHostUnit(t *testing.T) {
_, calls, output := runUpdateInstallEdited(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", func(script string) string {
// Drop archie-messaging from the list without touching its unit: the
// stale installed updater's view of the host.
return strings.Replace(script,
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging"`,
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"`,
1)
}, true)
if !strings.Contains(output, "archie-messaging.service") {
t.Errorf("refusal must name the unmanaged unit; output = %q", output)
}
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
// Nothing may be replaced on a host the update would leave partial.
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
}
// Carina's actual pair (archie-core-k94o): archie-messaging 1.35.0 beside an
// archie-state-store 1.30.0, the daemon at 1.30.0, and nothing approved to
// install. The component-driven updater reported only "nothing to install"
// while the two disagreed; now it reads each installed binary and names them.
func TestUpdateInstallRefusesWhenInstalledBinariesDisagree(t *testing.T) {
_, calls, output := runUpdateInstallFull(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
"archie-messaging": "1.35.0",
"archie-state-store": "1.30.0",
}, nil, true)
if !strings.Contains(output, "archie-messaging=1.35.0") {
t.Errorf("refusal must name the skewed binary and its version; output = %q", output)
}
if strings.Contains(output, "archie-state-store=") {
t.Errorf("refusal named a binary that agrees; output = %q", output)
}
assertCallAbsent(t, calls, "go build")
assertCallAbsent(t, calls, "install -m755")
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
}
// A binary built outside a release reports "dev"; on a released host that is
// skew too, and it must be named rather than read as agreement.
func TestUpdateInstallRefusesAnUnstampedBinary(t *testing.T) {
_, _, output := runUpdateInstallFull(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
"archie-ui": "dev",
}, nil, true)
if !strings.Contains(output, "archie-ui=dev") {
t.Errorf("refusal must name the unstamped binary; output = %q", output)
}
}
// The deployment is several processes, so the watchdog must cycle all of them,
// and in dependency order: the State Store owns archie.db and everything dials
// it, so it starts first and stops last (archie-core-hbqk).
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
t.Fatal(err)
}
work := t.TempDir()
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
if err := os.MkdirAll(binDir, 0o755); err != nil {
t.Fatal(err)
}
// archie-ui is new in this release: it has no .prev, so rolling back must
// remove it rather than leave a binary from a release that was withdrawn.
for name, content := range map[string]string{
"archied": "new", "archied.prev": "old",
"archie-state-store": "new", "archie-state-store.prev": "old",
"archie-ui": "new",
} {
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
t.Fatal(err)
}
}
callsPath := filepath.Join(work, "calls")
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
"ARCHIE_UPDATE_COMPONENTS=daemon",
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
calls, err := os.ReadFile(callsPath)
if err != nil {
t.Fatal(err)
}
indexOf := func(want string) int {
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
if line == want {
return i
}
}
return -1
}
startStore := indexOf("systemctl --user restart archie-state-store.service")
startDaemon := indexOf("systemctl --user restart archied.service")
stopUI := indexOf("systemctl --user stop archie-ui.service")
stopStore := indexOf("systemctl --user stop archie-state-store.service")
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
t.Errorf("State Store must start before archied; calls =\n%s", calls)
}
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
}
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
}
for _, name := range []string{"archied", "archie-state-store"} {
got, err := os.ReadFile(filepath.Join(binDir, name))
if err != nil {
t.Fatal(err)
}
if string(got) != "old" {
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
}
}
}
// The sidecar is the only record of the managed agent version, and
// archie-update-check refuses anything but a bare release version, so a stray
// escape in the write leaves the installed agent reported as unknown forever.
func TestUpdateInstallWritesABareAgentVersionSidecar(t *testing.T) {
versionFile := filepath.Join(t.TempDir(), "archie-agent.version")
runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
"ARCHIE_AGENT_VERSION_FILE": versionFile,
})
recorded, err := os.ReadFile(versionFile)
if err != nil {
t.Fatal(err)
}
if string(recorded) != "1.10.0\n" {
t.Fatalf("sidecar = %q, want %q", recorded, "1.10.0\n")
}
}
// freeAddress reserves a loopback address nothing is listening on and releases
// it, so a case can hand the watchdog a State Store target whose port stays
// closed until the fake State Store binds it.
func freeAddress(t *testing.T) string {
t.Helper()
var listen net.ListenConfig
listener, err := listen.Listen(t.Context(), "tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
address := listener.Addr().String()
if err := listener.Close(); err != nil {
t.Fatal(err)
}
return address
}
// TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied pins why an
// ordered start is not enough on its own.
//
// `systemctl --user start` returns once the process is forked, not once its port
// accepts. On a real 1.37.1 -> 1.38.0 update the State Store's unit reported
// started while it was still binding, archied started next, dialled a closed
// port, exited 1, and was rescued only by systemd's Restart=on-failure seven
// seconds later -- behind a "build and install succeeded" line, on a unit that
// need not carry that restart policy at all.
//
// The fake State Store binds its port two seconds AFTER systemctl returns, which
// is the race. The fake systemctl for archied records whether the port was open
// at the moment it was asked to start, so a watchdog that starts archied without
// waiting says so in the call log rather than passing quietly.
func TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied(t *testing.T) {
target := freeAddress(t)
_, _, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
curl: `exit 1`,
systemctl: `
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
case "$*" in
*"restart archie-state-store.service")
# Bind two seconds late, then hold the port: the State Store's unit returning
# before its listener exists is the whole failure being guarded against.
( sleep 2; exec python3 -c "import os,socket,time; h,p=os.environ['ARCHIE_UPDATE_STATE_STORE_TARGET'].rsplit(':',1); s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind((h,int(p))); s.listen(8); time.sleep(30)" ) >/dev/null 2>&1 &
;;
*"restart archied.service")
if ! (exec 3<>"/dev/tcp/${ARCHIE_UPDATE_STATE_STORE_TARGET%:*}/${ARCHIE_UPDATE_STATE_STORE_TARGET##*:}") 2>/dev/null; then
printf '%s\n' 'archied started before the state store accepted' >> "$ARCHIE_TEST_CALLS"
fi
;;
esac
exit 0`,
env: map[string]string{
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0",
"ARCHIE_UPDATE_UNITS": "archie-state-store archie-gateway archied",
"ARCHIE_UPDATE_STATE_STORE_TARGET": target,
"ARCHIE_UPDATE_STATE_STORE_TIMEOUT": "15",
},
})
assertCallContains(t, calls, "restart archie-state-store.service")
assertCallContains(t, calls, "restart archie-gateway.service")
assertCallContains(t, calls, "restart archied.service")
assertCallAbsent(t, calls, "archied started before the state store accepted")
}