The update check still read the retired archie/v* tag stream, so a host on 1.41.1 was offered runtime 1.35.0, and the installer built the agent image from that stale tag. Both now use the release's archied/v* tag. The dist-zip build also stamps runtimeVersion, so /version stops reporting 'dev'.
860 lines
33 KiB
Go
860 lines
33 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
|
|
assertCallContains(t, calls, "fetch --quiet --tags origin")
|
|
assertCallAbsent(t, calls, "git pull")
|
|
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
|
|
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallAbsent(t, calls, "docker compose build agent")
|
|
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
|
|
t.Fatalf("installed daemon = %q, want 1.13.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentAgent]; found {
|
|
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
|
|
}
|
|
if !result.RestartRequested {
|
|
t.Fatal("RestartRequested = false, want true for gateway update")
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallAbsent(t, calls, "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
|
|
t.Fatalf("installed agent = %q, want 1.10.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentDaemon]; found {
|
|
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallContains(t, calls, "go build", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
}
|
|
|
|
// Components are independently versioned (RELEASING.md), so the newest
|
|
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
|
|
// releases are the norm. Each changed component must be built from its own
|
|
// approved tag; forcing both tags onto one commit refuses every update once
|
|
// the two components' newest releases diverge.
|
|
// One version covers every component, so the managed worker image is built
|
|
// from the release's own archied/v* tag -- never from a stale archie/v* tag.
|
|
func TestUpdateInstallBuildsTheAgentImageFromTheReleaseTag(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.13.0",
|
|
})
|
|
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.13.0" {
|
|
t.Fatalf("installed = %#v, want the release version for both", result.Installed)
|
|
}
|
|
assertCallAbsent(t, calls, "archie-v1.13.0")
|
|
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
|
|
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
|
|
lastCheckout := -1
|
|
for i, call := range calls {
|
|
if strings.Contains(call, "checkout --quiet --detach") && i < agentImageBuild {
|
|
lastCheckout = i
|
|
}
|
|
}
|
|
if daemonBuild < 0 || agentImageBuild < 0 || lastCheckout < 0 {
|
|
t.Fatalf("missing expected step: daemon build %d, agent image build %d, checkout %d", daemonBuild, agentImageBuild, lastCheckout)
|
|
}
|
|
if !strings.Contains(calls[lastCheckout], "archied-v1.13.0") {
|
|
t.Errorf("image built after checking out %q, want the archied/v1.13.0 release; calls = %#v", calls[lastCheckout], calls)
|
|
}
|
|
}
|
|
|
|
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
|
|
t.Helper()
|
|
for i, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
|
|
assertCallAbsent(t, calls, forbidden)
|
|
}
|
|
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
|
|
t.Fatalf("no-op result = %#v, want empty result without restart", result)
|
|
}
|
|
}
|
|
|
|
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
|
|
// the two: systemd-run starts the watchdog with a clean environment, so a
|
|
// health address the daemon derived reaches the probe only if the install
|
|
// script hands it over explicitly (archie-core-1r4g).
|
|
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
|
|
})
|
|
|
|
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
|
|
}
|
|
|
|
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
components string
|
|
wantDaemon string
|
|
wantImageRollback bool
|
|
}{
|
|
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
|
|
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
|
|
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
|
|
if daemon != tt.wantDaemon {
|
|
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
|
|
}
|
|
if tt.wantImageRollback {
|
|
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
|
|
} else {
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
if !report.RolledBack || report.HealthCheck != "failed" {
|
|
t.Fatalf("report = %#v, want failed rollback", report)
|
|
}
|
|
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
|
|
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
|
|
}
|
|
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
|
|
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
|
|
// no other test reaches: a daemon that comes back up. Every other watchdog
|
|
// test fakes curl as an immediate failure, so before this one the probe URL
|
|
// the script actually builds was never dialled and its default could point
|
|
// anywhere (archie-core-1r4g).
|
|
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
|
daemon := httptest.NewServer(mux)
|
|
defer daemon.Close()
|
|
|
|
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
env: map[string]string{
|
|
"ARCHIE_HEALTH_URL": daemon.URL,
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
|
|
},
|
|
})
|
|
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
if installed != "new daemon" {
|
|
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
|
|
}
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
|
|
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
|
|
// script uses when nothing exports one. It has to be the address archied
|
|
// itself serves /healthz on ([health] listen); anything else times out and
|
|
// rolls back a healthy release.
|
|
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
|
|
report, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
|
|
})
|
|
|
|
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
}
|
|
|
|
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
|
t.Helper()
|
|
result, calls, _ := runUpdateInstall(t, environment, false)
|
|
return result, calls
|
|
}
|
|
|
|
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
|
|
// returns the parsed result (empty when the run was expected to fail), the
|
|
// recorded calls, and the combined output.
|
|
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
|
|
}
|
|
|
|
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
|
|
// file under the caller's control. Config-derived behaviour -- which database
|
|
// the updater backs up, which image it rebuilds -- is otherwise untestable,
|
|
// which is how the task-database backup path came to be wrong.
|
|
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallEdited(t, environment, configBody, nil, wantErr)
|
|
}
|
|
|
|
// runUpdateInstallEdited is runUpdateInstallWithConfig with the ability to edit
|
|
// the installer's text before it runs. The service list is hardcoded, so the
|
|
// only way to test the coverage guard against the stale list it exists for is to
|
|
// change that text -- which is exactly the state the guard must catch.
|
|
func runUpdateInstallEdited(t *testing.T, environment map[string]string, configBody string, edit func(string) string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallFull(t, environment, configBody, nil, edit, wantErr)
|
|
}
|
|
|
|
// runUpdateInstallFull is runUpdateInstallEdited with control over the versions
|
|
// already installed in the bin directory, so a skewed host can be built on
|
|
// purpose (archie-core-k94o).
|
|
func runUpdateInstallFull(t *testing.T, environment map[string]string, configBody string, installed map[string]string, edit func(string) string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
configPath := filepath.Join(work, "config.toml")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
callsPath := filepath.Join(work, "calls")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previousVersion := environment["ARCHIE_UPDATE_DAEMON_PREVIOUS"]
|
|
if previousVersion == "" {
|
|
previousVersion = "dev"
|
|
}
|
|
writeFakeBinary(t, binDir, "archied", previousVersion)
|
|
if err := os.WriteFile(filepath.Join(binDir, "archie-agent"), []byte("old archie-agent"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, version := range installed {
|
|
writeFakeBinary(t, binDir, name, version)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "git", `
|
|
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"clone "*)
|
|
source_dir="${@: -1}"
|
|
mkdir -p "$source_dir"
|
|
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
|
|
for c in archied archie-gateway archie-state-store archie-ui archie-messaging archie-playbooks archie-agent; do
|
|
mkdir -p "$source_dir/cmd/$c"
|
|
done
|
|
: > "$source_dir/docker-compose.yml"
|
|
;;
|
|
*"rev-parse --verify refs/tags/"*)
|
|
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
|
|
echo "${tag//\//-}"
|
|
;;
|
|
esac
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "go", `
|
|
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
|
# The pre-flight database snapshot runs through the release's own
|
|
# archie-state-store subcommand, so a release whose recovery path cannot run
|
|
# must refuse the update.
|
|
if [ "$1" = "run" ]; then
|
|
[ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && exit 1
|
|
exit 0
|
|
fi
|
|
out=""
|
|
name=""
|
|
version="dev"
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-o" ]; then out="$2"; fi
|
|
case "$1" in
|
|
./cmd/*) name="${1#./cmd/}" ;;
|
|
*buildinfo.Version=*)
|
|
version="${1##*buildinfo.Version=}"
|
|
version="${version%% *}"
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
mkdir -p "$(dirname "$out")"
|
|
# The installed binary must answer -version the way a real one does, or the
|
|
# updater's skew check cannot read it.
|
|
printf '#!/usr/bin/env bash\necho "%s %s"\n' "$name" "$version" > "$out"
|
|
chmod 755 "$out"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "docker", `
|
|
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$1" = image ] && [ "$2" = save ]; then
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = --output ]; then
|
|
: > "$2"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "systemctl", `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$2" = list-unit-files ]; then
|
|
# The reference deployment runs exactly these five as units. A command with
|
|
# no unit -- archie-playbooks, archie-agent -- has to look unitless, or the
|
|
# coverage guard would mistake every cmd/ directory for a host service.
|
|
present="${ARCHIE_TEST_PRESENT_UNITS:-archied.service archie-gateway.service archie-state-store.service archie-ui.service archie-messaging.service}"
|
|
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
|
|
*" $3 "*) exit 0 ;;
|
|
esac
|
|
case " $present " in
|
|
*" $3 "*) printf '%s enabled enabled\n' "$3" ;;
|
|
esac
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "install", `
|
|
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/install "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "cp", `
|
|
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/cp "$@"
|
|
`)
|
|
// The production adapter prepends /usr/local/go/bin. Run an otherwise
|
|
// byte-for-byte copy with the harness PATH left first so fake go records
|
|
// build intent without compiling or touching Docker/systemd.
|
|
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
script := string(scriptData)
|
|
if edit != nil {
|
|
script = edit(script)
|
|
}
|
|
scriptPath := filepath.Join(work, "archie-update-install")
|
|
script = strings.Replace(script, `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1)
|
|
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.CommandContext(ctx, scriptPath)
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
|
|
"ARCHIE_TEST_SOURCE_DIR="+root,
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_CONFIG_PATH="+configPath,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
)
|
|
for key, value := range environment {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
switch {
|
|
case wantErr && err == nil:
|
|
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
|
|
case !wantErr && err != nil:
|
|
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
if wantErr {
|
|
return Result{}, calls, string(output)
|
|
}
|
|
var result Result
|
|
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
|
found := false
|
|
for scanner.Scan() {
|
|
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
|
|
found = true
|
|
if err := json.Unmarshal([]byte(payload), &result); err != nil {
|
|
t.Fatalf("decode result %q: %v", payload, err)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("installer output has no result sentinel:\n%s", output)
|
|
}
|
|
return result, calls, string(output)
|
|
}
|
|
|
|
func writeFakeBinary(t *testing.T, dir, name, version string) {
|
|
t.Helper()
|
|
body := "#!/usr/bin/env bash\necho " + strconv.Quote(name+" "+version) + "\n"
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func writeFakeCommand(t *testing.T, dir, name, body string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(dir, name)
|
|
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// watchdogRun configures one end-to-end run of the watchdog script. curl is
|
|
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
|
|
// which is what turns a health probe into a genuine test of the URL the
|
|
// watchdog builds rather than of a stub that ignores it.
|
|
type watchdogRun struct {
|
|
components string
|
|
curl string
|
|
// systemctl replaces the fake systemctl on PATH, so a case can observe or
|
|
// condition what the watchdog does between units. Empty keeps the inert
|
|
// `exit 0` stub.
|
|
systemctl string
|
|
env map[string]string
|
|
}
|
|
|
|
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
|
|
t.Helper()
|
|
return runUpdateWatchdog(t, watchdogRun{
|
|
components: components,
|
|
curl: `exit 1`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
|
|
})
|
|
}
|
|
|
|
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := map[string]string{
|
|
"archied": "new daemon", "archied.prev": "old daemon",
|
|
"archie-agent-image.prev.tar": "saved old image",
|
|
}
|
|
for name, content := range files {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
systemctl := run.systemctl
|
|
if systemctl == "" {
|
|
systemctl = `exit 0`
|
|
}
|
|
writeFakeCommand(t, fakeDir, "systemctl", systemctl)
|
|
if run.curl != "" {
|
|
writeFakeCommand(t, fakeDir, "curl", run.curl)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
callsPath := filepath.Join(work, "calls")
|
|
|
|
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
|
|
"ARCHIE_UPDATE_COMPONENTS="+run.components,
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
|
|
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
|
|
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
|
|
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
|
|
)
|
|
for key, value := range run.env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
reportData, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report Report
|
|
if err := json.Unmarshal(reportData, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
return report, string(daemon), calls
|
|
}
|
|
|
|
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
|
|
}
|
|
|
|
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
if strings.Contains(call, fragment) {
|
|
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An archied release is five processes. Installing the daemon binary alone
|
|
// left the State Store, Gateway and UI on the previous release, so archied
|
|
// came up against a State Store nothing had started, failed its health check
|
|
// and rolled back (archie-core-hbqk).
|
|
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
})
|
|
|
|
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-messaging", "archie-playbooks"} {
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
|
|
assertCallContains(t, calls, "install -m755", "/"+cmd)
|
|
}
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
// The watchdog cannot restart what it is not told about.
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui archie-messaging")
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-messaging archie-playbooks")
|
|
}
|
|
|
|
// Installing binaries for processes the host has no unit for produces exactly
|
|
// the failure this bug was: archied dials a State Store nothing starts. Refuse
|
|
// before touching the host instead, and say what is missing.
|
|
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
|
|
_, calls, output := runUpdateInstall(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-state-store.service") {
|
|
t.Errorf("refusal must name the missing unit; output = %q", output)
|
|
}
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
// Nothing may be replaced on a host it would leave broken.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// The other half of the same contract, and the failure that actually happened
|
|
// (archie-core-1faq): the installed updater's GATEWAY_SERVICES predated
|
|
// archie-messaging, so the update neither backed it up nor restarted it and a
|
|
// hand-installed binary kept serving under an otherwise updated host. A unit
|
|
// this host runs must be named by the list, or the update refuses and says which.
|
|
func TestUpdateInstallRefusesWhenTheServiceListOmitsAHostUnit(t *testing.T) {
|
|
_, calls, output := runUpdateInstallEdited(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", func(script string) string {
|
|
// Drop archie-messaging from the list without touching its unit: the
|
|
// stale installed updater's view of the host.
|
|
return strings.Replace(script,
|
|
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging"`,
|
|
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"`,
|
|
1)
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-messaging.service") {
|
|
t.Errorf("refusal must name the unmanaged unit; output = %q", output)
|
|
}
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
// Nothing may be replaced on a host the update would leave partial.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// Carina's actual pair (archie-core-k94o): archie-messaging 1.35.0 beside an
|
|
// archie-state-store 1.30.0, the daemon at 1.30.0, and nothing approved to
|
|
// install. The component-driven updater reported only "nothing to install"
|
|
// while the two disagreed; now it reads each installed binary and names them.
|
|
func TestUpdateInstallRefusesWhenInstalledBinariesDisagree(t *testing.T) {
|
|
_, calls, output := runUpdateInstallFull(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
|
|
"archie-messaging": "1.35.0",
|
|
"archie-state-store": "1.30.0",
|
|
}, nil, true)
|
|
|
|
if !strings.Contains(output, "archie-messaging=1.35.0") {
|
|
t.Errorf("refusal must name the skewed binary and its version; output = %q", output)
|
|
}
|
|
if strings.Contains(output, "archie-state-store=") {
|
|
t.Errorf("refusal named a binary that agrees; output = %q", output)
|
|
}
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
}
|
|
|
|
// A binary built outside a release reports "dev"; on a released host that is
|
|
// skew too, and it must be named rather than read as agreement.
|
|
func TestUpdateInstallRefusesAnUnstampedBinary(t *testing.T) {
|
|
_, _, output := runUpdateInstallFull(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
|
|
"archie-ui": "dev",
|
|
}, nil, true)
|
|
|
|
if !strings.Contains(output, "archie-ui=dev") {
|
|
t.Errorf("refusal must name the unstamped binary; output = %q", output)
|
|
}
|
|
}
|
|
|
|
// The deployment is several processes, so the watchdog must cycle all of them,
|
|
// and in dependency order: the State Store owns archie.db and everything dials
|
|
// it, so it starts first and stops last (archie-core-hbqk).
|
|
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// archie-ui is new in this release: it has no .prev, so rolling back must
|
|
// remove it rather than leave a binary from a release that was withdrawn.
|
|
for name, content := range map[string]string{
|
|
"archied": "new", "archied.prev": "old",
|
|
"archie-state-store": "new", "archie-state-store.prev": "old",
|
|
"archie-ui": "new",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
|
"ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
indexOf := func(want string) int {
|
|
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
|
if line == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
startStore := indexOf("systemctl --user restart archie-state-store.service")
|
|
startDaemon := indexOf("systemctl --user restart archied.service")
|
|
stopUI := indexOf("systemctl --user stop archie-ui.service")
|
|
stopStore := indexOf("systemctl --user stop archie-state-store.service")
|
|
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
|
|
t.Errorf("State Store must start before archied; calls =\n%s", calls)
|
|
}
|
|
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
|
|
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
|
|
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-state-store"} {
|
|
got, err := os.ReadFile(filepath.Join(binDir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(got) != "old" {
|
|
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The sidecar is the only record of the managed agent version, and
|
|
// archie-update-check refuses anything but a bare release version, so a stray
|
|
// escape in the write leaves the installed agent reported as unknown forever.
|
|
func TestUpdateInstallWritesABareAgentVersionSidecar(t *testing.T) {
|
|
versionFile := filepath.Join(t.TempDir(), "archie-agent.version")
|
|
runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
"ARCHIE_AGENT_VERSION_FILE": versionFile,
|
|
})
|
|
|
|
recorded, err := os.ReadFile(versionFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(recorded) != "1.10.0\n" {
|
|
t.Fatalf("sidecar = %q, want %q", recorded, "1.10.0\n")
|
|
}
|
|
}
|
|
|
|
// freeAddress reserves a loopback address nothing is listening on and releases
|
|
// it, so a case can hand the watchdog a State Store target whose port stays
|
|
// closed until the fake State Store binds it.
|
|
func freeAddress(t *testing.T) string {
|
|
t.Helper()
|
|
var listen net.ListenConfig
|
|
listener, err := listen.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
address := listener.Addr().String()
|
|
if err := listener.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return address
|
|
}
|
|
|
|
// TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied pins why an
|
|
// ordered start is not enough on its own.
|
|
//
|
|
// `systemctl --user start` returns once the process is forked, not once its port
|
|
// accepts. On a real 1.37.1 -> 1.38.0 update the State Store's unit reported
|
|
// started while it was still binding, archied started next, dialled a closed
|
|
// port, exited 1, and was rescued only by systemd's Restart=on-failure seven
|
|
// seconds later -- behind a "build and install succeeded" line, on a unit that
|
|
// need not carry that restart policy at all.
|
|
//
|
|
// The fake State Store binds its port two seconds AFTER systemctl returns, which
|
|
// is the race. The fake systemctl for archied records whether the port was open
|
|
// at the moment it was asked to start, so a watchdog that starts archied without
|
|
// waiting says so in the call log rather than passing quietly.
|
|
func TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied(t *testing.T) {
|
|
target := freeAddress(t)
|
|
_, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `exit 1`,
|
|
systemctl: `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"restart archie-state-store.service")
|
|
# Bind two seconds late, then hold the port: the State Store's unit returning
|
|
# before its listener exists is the whole failure being guarded against.
|
|
( sleep 2; exec python3 -c "import os,socket,time; h,p=os.environ['ARCHIE_UPDATE_STATE_STORE_TARGET'].rsplit(':',1); s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind((h,int(p))); s.listen(8); time.sleep(30)" ) >/dev/null 2>&1 &
|
|
;;
|
|
*"restart archied.service")
|
|
if ! (exec 3<>"/dev/tcp/${ARCHIE_UPDATE_STATE_STORE_TARGET%:*}/${ARCHIE_UPDATE_STATE_STORE_TARGET##*:}") 2>/dev/null; then
|
|
printf '%s\n' 'archied started before the state store accepted' >> "$ARCHIE_TEST_CALLS"
|
|
fi
|
|
;;
|
|
esac
|
|
exit 0`,
|
|
env: map[string]string{
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0",
|
|
"ARCHIE_UPDATE_UNITS": "archie-state-store archie-gateway archied",
|
|
"ARCHIE_UPDATE_STATE_STORE_TARGET": target,
|
|
"ARCHIE_UPDATE_STATE_STORE_TIMEOUT": "15",
|
|
},
|
|
})
|
|
|
|
assertCallContains(t, calls, "restart archie-state-store.service")
|
|
assertCallContains(t, calls, "restart archie-gateway.service")
|
|
assertCallContains(t, calls, "restart archied.service")
|
|
assertCallAbsent(t, calls, "archied started before the state store accepted")
|
|
}
|