Components are independently versioned (RELEASING.md), so the newest archied/v* and archie/v* tags usually sit on different commits -- gateway-only releases are the norm. The installer required both tags to resolve to the same commit whenever both components changed, which refused every update once the two newest releases diverged (archied/v1.24.0 at8950bde, archie/v1.23.0 at6cf73db). Build each changed component from its own tag's commit instead: daemon binaries and the adapter refresh from the daemon release, the managed worker image from the agent release.
833 lines
32 KiB
Go
833 lines
32 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
|
|
assertCallContains(t, calls, "fetch --quiet --tags origin")
|
|
assertCallAbsent(t, calls, "git pull")
|
|
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
|
|
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallAbsent(t, calls, "docker compose build agent")
|
|
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
|
|
t.Fatalf("installed daemon = %q, want 1.13.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentAgent]; found {
|
|
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
|
|
}
|
|
if !result.RestartRequested {
|
|
t.Fatal("RestartRequested = false, want true for gateway update")
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallAbsent(t, calls, "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
|
|
t.Fatalf("installed agent = %q, want 1.10.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentDaemon]; found {
|
|
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallContains(t, calls, "go build", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
}
|
|
|
|
// Components are independently versioned (RELEASING.md), so the newest
|
|
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
|
|
// releases are the norm. Each changed component must be built from its own
|
|
// approved tag; forcing both tags onto one commit refuses every update once
|
|
// the two components' newest releases diverge.
|
|
func TestUpdateInstallBuildsEachComponentFromItsOwnTag(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
// The derived fake commits differ per tag, so these assertions fail if
|
|
// either component is built from the other's release.
|
|
daemonCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archied-v1.13.0")
|
|
agentCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archie-v1.10.0")
|
|
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
|
|
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
|
|
if daemonCheckout < 0 || agentCheckout < 0 || daemonBuild < 0 || agentImageBuild < 0 {
|
|
t.Fatalf("missing expected step: daemon checkout %d, daemon build %d, agent checkout %d, agent image build %d",
|
|
daemonCheckout, daemonBuild, agentCheckout, agentImageBuild)
|
|
}
|
|
if daemonCheckout > daemonBuild {
|
|
t.Errorf("daemon binaries must be built from the daemon tag's commit; calls = %#v", calls)
|
|
}
|
|
if agentCheckout > agentImageBuild {
|
|
t.Errorf("the managed worker image must be built from the agent tag's commit; calls = %#v", calls)
|
|
}
|
|
}
|
|
|
|
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
|
|
t.Helper()
|
|
for i, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
|
|
assertCallAbsent(t, calls, forbidden)
|
|
}
|
|
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
|
|
t.Fatalf("no-op result = %#v, want empty result without restart", result)
|
|
}
|
|
}
|
|
|
|
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
|
|
// the two: systemd-run starts the watchdog with a clean environment, so a
|
|
// health address the daemon derived reaches the probe only if the install
|
|
// script hands it over explicitly (archie-core-1r4g).
|
|
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
|
|
})
|
|
|
|
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
|
|
}
|
|
|
|
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
components string
|
|
wantDaemon string
|
|
wantImageRollback bool
|
|
}{
|
|
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
|
|
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
|
|
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
|
|
if daemon != tt.wantDaemon {
|
|
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
|
|
}
|
|
if tt.wantImageRollback {
|
|
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
|
|
} else {
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
if !report.RolledBack || report.HealthCheck != "failed" {
|
|
t.Fatalf("report = %#v, want failed rollback", report)
|
|
}
|
|
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
|
|
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
|
|
}
|
|
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
|
|
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
|
|
// no other test reaches: a daemon that comes back up. Every other watchdog
|
|
// test fakes curl as an immediate failure, so before this one the probe URL
|
|
// the script actually builds was never dialled and its default could point
|
|
// anywhere (archie-core-1r4g).
|
|
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
|
daemon := httptest.NewServer(mux)
|
|
defer daemon.Close()
|
|
|
|
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
env: map[string]string{
|
|
"ARCHIE_HEALTH_URL": daemon.URL,
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
|
|
},
|
|
})
|
|
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
if installed != "new daemon" {
|
|
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
|
|
}
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
|
|
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
|
|
// script uses when nothing exports one. It has to be the address archied
|
|
// itself serves /healthz on ([health] listen); anything else times out and
|
|
// rolls back a healthy release.
|
|
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
|
|
report, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
|
|
})
|
|
|
|
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
}
|
|
|
|
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
|
|
createSQLiteDatabase(t, path, version)
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
|
|
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
db, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
var version int
|
|
if err := db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if version != 1 {
|
|
t.Fatalf("restored schema version = %d, want 1", version)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
|
stopIndex, copyIndex, startIndex := -1, -1, -1
|
|
for i, line := range callLines {
|
|
switch line {
|
|
case "systemctl --user stop archied.service":
|
|
stopIndex = i
|
|
case "systemctl --user restart archied.service":
|
|
startIndex = i
|
|
}
|
|
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
|
|
copyIndex = i
|
|
}
|
|
}
|
|
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
|
|
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
|
|
}
|
|
if startIndex < 0 || startIndex <= copyIndex {
|
|
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
|
|
}
|
|
}
|
|
|
|
func createSQLiteDatabase(t *testing.T, path string, version int) {
|
|
t.Helper()
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
t.Fatalf("open database %s: %v", path, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.ExecContext(t.Context(), "PRAGMA user_version = "+strconv.Itoa(version)); err != nil {
|
|
t.Fatalf("set database version for %s: %v", path, err)
|
|
}
|
|
if _, err := db.ExecContext(t.Context(), "CREATE TABLE tasks (id INTEGER)"); err != nil {
|
|
t.Fatalf("create tasks table in %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
|
t.Helper()
|
|
result, calls, _ := runUpdateInstall(t, environment, false)
|
|
return result, calls
|
|
}
|
|
|
|
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
|
|
// returns the parsed result (empty when the run was expected to fail), the
|
|
// recorded calls, and the combined output.
|
|
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
|
|
}
|
|
|
|
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
|
|
// file under the caller's control. Config-derived behaviour -- which database
|
|
// the updater backs up, which image it rebuilds -- is otherwise untestable,
|
|
// which is how the task-database backup path came to be wrong.
|
|
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
configPath := filepath.Join(work, "config.toml")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
callsPath := filepath.Join(work, "calls")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-agent"} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte("old "+name), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writeFakeCommand(t, fakeDir, "git", `
|
|
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"clone "*)
|
|
source_dir="${@: -1}"
|
|
mkdir -p "$source_dir"
|
|
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
|
|
for c in archied archie-gateway archie-state-store archie-ui archie-playbooks archie-agent; do
|
|
mkdir -p "$source_dir/cmd/$c"
|
|
done
|
|
: > "$source_dir/docker-compose.yml"
|
|
;;
|
|
*"rev-parse --verify refs/tags/"*)
|
|
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
|
|
echo "${tag//\//-}"
|
|
;;
|
|
esac
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "go", `
|
|
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
|
out=""
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-o" ]; then out="$2"; break; fi
|
|
shift
|
|
done
|
|
mkdir -p "$(dirname "$out")"
|
|
printf 'built\n' > "$out"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "docker", `
|
|
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$1" = image ] && [ "$2" = save ]; then
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = --output ]; then
|
|
: > "$2"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "systemctl", `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$2" = list-unit-files ]; then
|
|
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
|
|
*" $3 "*) exit 0 ;;
|
|
esac
|
|
printf '%s enabled enabled\n' "$3"
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "install", `
|
|
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/install "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "cp", `
|
|
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/cp "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "sqlite3", `
|
|
printf '%s\n' "sqlite3 $*" >> "$ARCHIE_TEST_CALLS"
|
|
[ "${ARCHIE_TEST_SQLITE3_FAILS:-}" = "1" ] && exit 1
|
|
# Reproduce ".backup '<path>'" by writing a plausible backup beside the
|
|
# source, so the watchdog's existence check behaves as it would in production.
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
.backup*) target="$(printf '%s' "$arg" | sed "s/^\.backup[[:space:]]*//; s/^'//; s/'$//")" ;;
|
|
esac
|
|
done
|
|
if [ -n "${target:-}" ]; then
|
|
mkdir -p "$(dirname "$target")"
|
|
printf 'backup\n' > "$target"
|
|
fi
|
|
`)
|
|
|
|
// The production adapter prepends /usr/local/go/bin. Run an otherwise
|
|
// byte-for-byte copy with the harness PATH left first so fake go records
|
|
// build intent without compiling or touching Docker/systemd.
|
|
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
scriptPath := filepath.Join(work, "archie-update-install")
|
|
scriptData = []byte(strings.Replace(string(scriptData), `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1))
|
|
if err := os.WriteFile(scriptPath, scriptData, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.CommandContext(ctx, scriptPath)
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
|
|
"ARCHIE_TEST_SOURCE_DIR="+root,
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_CONFIG_PATH="+configPath,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
)
|
|
for key, value := range environment {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
switch {
|
|
case wantErr && err == nil:
|
|
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
|
|
case !wantErr && err != nil:
|
|
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
if wantErr {
|
|
return Result{}, calls, string(output)
|
|
}
|
|
var result Result
|
|
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
|
found := false
|
|
for scanner.Scan() {
|
|
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
|
|
found = true
|
|
if err := json.Unmarshal([]byte(payload), &result); err != nil {
|
|
t.Fatalf("decode result %q: %v", payload, err)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("installer output has no result sentinel:\n%s", output)
|
|
}
|
|
return result, calls, string(output)
|
|
}
|
|
|
|
func writeFakeCommand(t *testing.T, dir, name, body string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(dir, name)
|
|
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// watchdogRun configures one end-to-end run of the watchdog script. curl is
|
|
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
|
|
// which is what turns a health probe into a genuine test of the URL the
|
|
// watchdog builds rather than of a stub that ignores it.
|
|
type watchdogRun struct {
|
|
components string
|
|
curl string
|
|
env map[string]string
|
|
}
|
|
|
|
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
|
|
t.Helper()
|
|
return runUpdateWatchdog(t, watchdogRun{
|
|
components: components,
|
|
curl: `exit 1`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
|
|
})
|
|
}
|
|
|
|
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := map[string]string{
|
|
"archied": "new daemon", "archied.prev": "old daemon",
|
|
"archie-agent-image.prev.tar": "saved old image",
|
|
}
|
|
for name, content := range files {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
|
|
if run.curl != "" {
|
|
writeFakeCommand(t, fakeDir, "curl", run.curl)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
callsPath := filepath.Join(work, "calls")
|
|
|
|
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
|
|
"ARCHIE_UPDATE_COMPONENTS="+run.components,
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
|
|
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
|
|
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
|
|
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
|
|
)
|
|
for key, value := range run.env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
reportData, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report Report
|
|
if err := json.Unmarshal(reportData, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
return report, string(daemon), calls
|
|
}
|
|
|
|
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
|
|
}
|
|
|
|
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
if strings.Contains(call, fragment) {
|
|
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An archied release is five processes. Installing the daemon binary alone
|
|
// left the State Store, Gateway and UI on the previous release, so archied
|
|
// came up against a State Store nothing had started, failed its health check
|
|
// and rolled back (archie-core-hbqk).
|
|
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
})
|
|
|
|
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-playbooks"} {
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
|
|
assertCallContains(t, calls, "install -m755", "/"+cmd)
|
|
}
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
// The watchdog cannot restart what it is not told about.
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui")
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-playbooks")
|
|
}
|
|
|
|
// Installing binaries for processes the host has no unit for produces exactly
|
|
// the failure this bug was: archied dials a State Store nothing starts. Refuse
|
|
// before touching the host instead, and say what is missing.
|
|
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
|
|
_, calls, output := runUpdateInstall(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-state-store.service") {
|
|
t.Errorf("refusal must name the missing unit; output = %q", output)
|
|
}
|
|
// Nothing may be replaced on a host it would leave broken.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// The deployment is several processes, so the watchdog must cycle all of them,
|
|
// and in dependency order: the State Store owns archie.db and everything dials
|
|
// it, so it starts first and stops last (archie-core-hbqk).
|
|
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// archie-ui is new in this release: it has no .prev, so rolling back must
|
|
// remove it rather than leave a binary from a release that was withdrawn.
|
|
for name, content := range map[string]string{
|
|
"archied": "new", "archied.prev": "old",
|
|
"archie-state-store": "new", "archie-state-store.prev": "old",
|
|
"archie-ui": "new",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
|
"ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
indexOf := func(want string) int {
|
|
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
|
if line == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
startStore := indexOf("systemctl --user restart archie-state-store.service")
|
|
startDaemon := indexOf("systemctl --user restart archied.service")
|
|
stopUI := indexOf("systemctl --user stop archie-ui.service")
|
|
stopStore := indexOf("systemctl --user stop archie-state-store.service")
|
|
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
|
|
t.Errorf("State Store must start before archied; calls =\n%s", calls)
|
|
}
|
|
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
|
|
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
|
|
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-state-store"} {
|
|
got, err := os.ReadFile(filepath.Join(binDir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(got) != "old" {
|
|
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The updater must back up the database the daemon actually uses, which is
|
|
// `<db_path>-tasks.sqlite` (taskDBPath in internal/app/archied/main.go), not
|
|
// the configured path itself. The configured path may be a zero-byte
|
|
// placeholder while the real store sits beside it -- so backing the literal
|
|
// value up produces an empty backup, and a rollback then restores that empty
|
|
// file over the database and deletes its -wal/-shm. A rollback that destroys
|
|
// the task store is worse than no rollback.
|
|
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
realStore := configured + "-tasks.sqlite"
|
|
if err := os.WriteFile(configured, nil, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(realStore, []byte("real task data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backup := filepath.Join(work, "backup.sqlite")
|
|
|
|
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TASK_DB_BACKUP": backup,
|
|
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
|
|
|
assertCallContains(t, calls, "sqlite3", realStore, ".backup")
|
|
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
|
|
// The zero-byte placeholder is not the store, and must never be the thing
|
|
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
|
|
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+configured+" ")
|
|
}
|
|
|
|
// db_path also appears in other sections -- [indexing] owns its own database.
|
|
// Reading the first match in the file can select an unrelated database, so the
|
|
// value must come from the daemon's own top-level key.
|
|
func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
realStore := configured + "-tasks.sqlite"
|
|
indexStore := filepath.Join(work, "workspace-indexes.db")
|
|
for path, content := range map[string]string{realStore: "real task data", indexStore: "index data"} {
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
backup := filepath.Join(work, "backup.sqlite")
|
|
|
|
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TASK_DB_BACKUP": backup,
|
|
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
|
|
|
assertCallContains(t, calls, "sqlite3", realStore)
|
|
for _, call := range calls {
|
|
if strings.Contains(call, "sqlite3") && strings.Contains(call, indexStore) {
|
|
t.Errorf("backed up the indexing database %s: %q", indexStore, call)
|
|
}
|
|
}
|
|
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+indexStore)
|
|
}
|
|
|
|
// A store that cannot be backed up must fail the update before anything is
|
|
// replaced. Installing with no usable rollback is how a failed update becomes
|
|
// a destroyed task store.
|
|
func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
if err := os.WriteFile(configured+"-tasks.sqlite", []byte("data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, calls, output := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_SQLITE3_FAILS": "1",
|
|
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
|
|
|
|
if !strings.Contains(output, "task database") {
|
|
t.Errorf("refusal must name the task database; output = %q", output)
|
|
}
|
|
// Nothing of the release may be placed on the host, and no backup of the
|
|
// failure path may be attempted either.
|
|
for _, call := range calls {
|
|
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
|
|
t.Errorf("refused update still installed %q", call)
|
|
}
|
|
}
|
|
assertCallAbsent(t, calls, "arhied.prev")
|
|
}
|