Without -db, backup/restore/validate/rollback resolve database_url from the config and work on the whole PostgreSQL database. Restore claims every service role; rollback claims the State Store role; validate refuses a schema newer than the binary. The installer refuses a daemon update it cannot snapshot first; the watchdog never restores the database and prints the explicit offline restore instead.
1064 lines
42 KiB
Go
1064 lines
42 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"bufio"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
func TestUpdateInstallGatewayOnlySkipsRuntimeWork(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
assertCallContains(t, calls, "git clone", "--no-checkout", "https://example.invalid/archie-core.git")
|
|
assertCallContains(t, calls, "fetch --quiet --tags origin")
|
|
assertCallAbsent(t, calls, "git pull")
|
|
assertCallContains(t, calls, "git -C", "rev-parse --verify refs/tags/archied/v1.13.0^{commit}")
|
|
assertCallContains(t, calls, "checkout --quiet --detach archied-v1.13.0")
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.runtimeVersion=1.9.9", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallAbsent(t, calls, "docker compose build agent")
|
|
if got := result.Installed[ComponentDaemon]; got != "1.13.0" {
|
|
t.Fatalf("installed daemon = %q, want 1.13.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentAgent]; found {
|
|
t.Fatalf("gateway-only result unexpectedly claims an agent install: %#v", result.Installed)
|
|
}
|
|
if !result.RestartRequested {
|
|
t.Fatal("RestartRequested = false, want true for gateway update")
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallRuntimeOnlySkipsGatewayBuild(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallAbsent(t, calls, "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "--build-arg RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if got := result.Installed[ComponentAgent]; got != "1.10.0" {
|
|
t.Fatalf("installed agent = %q, want 1.10.0", got)
|
|
}
|
|
if _, found := result.Installed[ComponentDaemon]; found {
|
|
t.Fatalf("runtime-only result unexpectedly claims a daemon install: %#v", result.Installed)
|
|
}
|
|
}
|
|
|
|
func TestUpdateInstallBothComponentsBuildsDaemonAndManagedWorkerImage(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
assertCallContains(t, calls, "go build", "./cmd/archied")
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
assertCallContains(t, calls, "docker image save", "registry.example/archie-agent:stable")
|
|
assertCallContains(t, calls, "docker build", "RUNTIME_VERSION=1.10.0", "--tag registry.example/archie-agent:stable")
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
}
|
|
|
|
// Components are independently versioned (RELEASING.md), so the newest
|
|
// archied/v* and archie/v* tags usually sit on different commits -- gateway-only
|
|
// releases are the norm. Each changed component must be built from its own
|
|
// approved tag; forcing both tags onto one commit refuses every update once
|
|
// the two components' newest releases diverge.
|
|
func TestUpdateInstallBuildsEachComponentFromItsOwnTag(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
})
|
|
|
|
if result.Installed[ComponentDaemon] != "1.13.0" || result.Installed[ComponentAgent] != "1.10.0" {
|
|
t.Fatalf("installed = %#v, want both release versions", result.Installed)
|
|
}
|
|
// The derived fake commits differ per tag, so these assertions fail if
|
|
// either component is built from the other's release.
|
|
daemonCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archied-v1.13.0")
|
|
agentCheckout := indexOfCallContaining(t, calls, "checkout --quiet --detach archie-v1.10.0")
|
|
daemonBuild := indexOfCallContaining(t, calls, "go build", "./cmd/archied")
|
|
agentImageBuild := indexOfCallContaining(t, calls, "docker build")
|
|
if daemonCheckout < 0 || agentCheckout < 0 || daemonBuild < 0 || agentImageBuild < 0 {
|
|
t.Fatalf("missing expected step: daemon checkout %d, daemon build %d, agent checkout %d, agent image build %d",
|
|
daemonCheckout, daemonBuild, agentCheckout, agentImageBuild)
|
|
}
|
|
if daemonCheckout > daemonBuild {
|
|
t.Errorf("daemon binaries must be built from the daemon tag's commit; calls = %#v", calls)
|
|
}
|
|
if agentCheckout > agentImageBuild {
|
|
t.Errorf("the managed worker image must be built from the agent tag's commit; calls = %#v", calls)
|
|
}
|
|
}
|
|
|
|
func indexOfCallContaining(t *testing.T, calls []string, fragments ...string) int {
|
|
t.Helper()
|
|
for i, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
|
|
result, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
})
|
|
|
|
for _, forbidden := range []string{"git pull", "go build", "docker ", "systemd-run ", "install ", "cp "} {
|
|
assertCallAbsent(t, calls, forbidden)
|
|
}
|
|
if len(result.Previous) != 0 || len(result.Installed) != 0 || len(result.Components) != 0 || result.RestartRequested {
|
|
t.Fatalf("no-op result = %#v, want empty result without restart", result)
|
|
}
|
|
}
|
|
|
|
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
|
|
// the two: systemd-run starts the watchdog with a clean environment, so a
|
|
// health address the daemon derived reaches the probe only if the install
|
|
// script hands it over explicitly (archie-core-1r4g).
|
|
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
|
|
})
|
|
|
|
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
|
|
}
|
|
|
|
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
components string
|
|
wantDaemon string
|
|
wantImageRollback bool
|
|
}{
|
|
{name: "gateway only", components: "daemon", wantDaemon: "old daemon"},
|
|
{name: "runtime only", components: "agent", wantDaemon: "new daemon", wantImageRollback: true},
|
|
{name: "both", components: "daemon,agent", wantDaemon: "old daemon", wantImageRollback: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
report, daemon, calls := runUpdateWatchdogFailure(t, tt.components)
|
|
if daemon != tt.wantDaemon {
|
|
t.Fatalf("daemon after rollback = %q, want %q", daemon, tt.wantDaemon)
|
|
}
|
|
if tt.wantImageRollback {
|
|
assertCallContains(t, calls, "docker image load", "archie-agent-image.prev.tar")
|
|
} else {
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
if !report.RolledBack || report.HealthCheck != "failed" {
|
|
t.Fatalf("report = %#v, want failed rollback", report)
|
|
}
|
|
if _, found := report.Installed[ComponentDaemon]; found != strings.Contains(tt.components, "daemon") {
|
|
t.Fatalf("daemon report membership = %v for components %q", found, tt.components)
|
|
}
|
|
if _, found := report.Installed[ComponentAgent]; found != strings.Contains(tt.components, "agent") {
|
|
t.Fatalf("agent report membership = %v for components %q", found, tt.components)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
|
|
// no other test reaches: a daemon that comes back up. Every other watchdog
|
|
// test fakes curl as an immediate failure, so before this one the probe URL
|
|
// the script actually builds was never dialled and its default could point
|
|
// anywhere (archie-core-1r4g).
|
|
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
|
daemon := httptest.NewServer(mux)
|
|
defer daemon.Close()
|
|
|
|
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
env: map[string]string{
|
|
"ARCHIE_HEALTH_URL": daemon.URL,
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
|
|
},
|
|
})
|
|
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
if installed != "new daemon" {
|
|
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
|
|
}
|
|
assertCallAbsent(t, calls, "docker image load")
|
|
}
|
|
|
|
// TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon pins the fallback the
|
|
// script uses when nothing exports one. It has to be the address archied
|
|
// itself serves /healthz on ([health] listen); anything else times out and
|
|
// rolls back a healthy release.
|
|
func TestUpdateWatchdogDefaultHealthURLTargetsTheDaemon(t *testing.T) {
|
|
report, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
|
|
})
|
|
|
|
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8485/healthz")
|
|
if report.HealthCheck != "passed" || report.RolledBack {
|
|
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
|
}
|
|
}
|
|
|
|
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
dbPath, backupPath := filepath.Join(work, "tasks.sqlite"), filepath.Join(work, "tasks.prev.sqlite")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, content := range map[string]string{"archied": "new", "archied.prev": "old"} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for path, version := range map[string]int{dbPath: 2, backupPath: 1} {
|
|
createSQLiteDatabase(t, path, version)
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "cp", `printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"; /usr/bin/cp "$@"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", "ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.19.10", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.20.0",
|
|
"ARCHIE_TASK_DB_PATH="+dbPath, "ARCHIE_TASK_DB_BACKUP="+backupPath)
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
db, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
var version int
|
|
if err := db.QueryRowContext(t.Context(), "PRAGMA user_version").Scan(&version); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if version != 1 {
|
|
t.Fatalf("restored schema version = %d, want 1", version)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callLines := strings.Split(strings.TrimSpace(string(calls)), "\n")
|
|
stopIndex, copyIndex, startIndex := -1, -1, -1
|
|
for i, line := range callLines {
|
|
switch line {
|
|
case "systemctl --user stop archied.service":
|
|
stopIndex = i
|
|
case "systemctl --user restart archied.service":
|
|
startIndex = i
|
|
}
|
|
if strings.HasPrefix(line, "cp ") && strings.Contains(line, "tasks.prev.sqlite") {
|
|
copyIndex = i
|
|
}
|
|
}
|
|
if stopIndex < 0 || copyIndex < 0 || stopIndex > copyIndex {
|
|
t.Fatalf("rollback must stop service before restoring database; calls = %q", calls)
|
|
}
|
|
if startIndex < 0 || startIndex <= copyIndex {
|
|
t.Fatalf("rollback must restart service after restoring database; calls = %q", calls)
|
|
}
|
|
}
|
|
|
|
func createSQLiteDatabase(t *testing.T, path string, version int) {
|
|
t.Helper()
|
|
db, err := sql.Open("sqlite", path)
|
|
if err != nil {
|
|
t.Fatalf("open database %s: %v", path, err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.ExecContext(t.Context(), "PRAGMA user_version = "+strconv.Itoa(version)); err != nil {
|
|
t.Fatalf("set database version for %s: %v", path, err)
|
|
}
|
|
if _, err := db.ExecContext(t.Context(), "CREATE TABLE tasks (id INTEGER)"); err != nil {
|
|
t.Fatalf("create tasks table in %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
func runUpdateInstallScript(t *testing.T, environment map[string]string) (Result, []string) {
|
|
t.Helper()
|
|
result, calls, _ := runUpdateInstall(t, environment, false)
|
|
return result, calls
|
|
}
|
|
|
|
// runUpdateInstall runs the adapter, optionally expecting it to refuse. It
|
|
// returns the parsed result (empty when the run was expected to fail), the
|
|
// recorded calls, and the combined output.
|
|
func runUpdateInstall(t *testing.T, environment map[string]string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallWithConfig(t, environment, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", wantErr)
|
|
}
|
|
|
|
// runUpdateInstallWithConfig is runUpdateInstall with the operator's config
|
|
// file under the caller's control. Config-derived behaviour -- which database
|
|
// the updater backs up, which image it rebuilds -- is otherwise untestable,
|
|
// which is how the task-database backup path came to be wrong.
|
|
func runUpdateInstallWithConfig(t *testing.T, environment map[string]string, configBody string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallEdited(t, environment, configBody, nil, wantErr)
|
|
}
|
|
|
|
// runUpdateInstallEdited is runUpdateInstallWithConfig with the ability to edit
|
|
// the installer's text before it runs. The service list is hardcoded, so the
|
|
// only way to test the coverage guard against the stale list it exists for is to
|
|
// change that text -- which is exactly the state the guard must catch.
|
|
func runUpdateInstallEdited(t *testing.T, environment map[string]string, configBody string, edit func(string) string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
return runUpdateInstallFull(t, environment, configBody, nil, edit, wantErr)
|
|
}
|
|
|
|
// runUpdateInstallFull is runUpdateInstallEdited with control over the versions
|
|
// already installed in the bin directory, so a skewed host can be built on
|
|
// purpose (archie-core-k94o).
|
|
func runUpdateInstallFull(t *testing.T, environment map[string]string, configBody string, installed map[string]string, edit func(string) string, wantErr bool) (Result, []string, string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
configPath := filepath.Join(work, "config.toml")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
callsPath := filepath.Join(work, "calls")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previousVersion := environment["ARCHIE_UPDATE_DAEMON_PREVIOUS"]
|
|
if previousVersion == "" {
|
|
previousVersion = "dev"
|
|
}
|
|
writeFakeBinary(t, binDir, "archied", previousVersion)
|
|
if err := os.WriteFile(filepath.Join(binDir, "archie-agent"), []byte("old archie-agent"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, version := range installed {
|
|
writeFakeBinary(t, binDir, name, version)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "git", `
|
|
printf '%s\n' "git $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"clone "*)
|
|
source_dir="${@: -1}"
|
|
mkdir -p "$source_dir"
|
|
/usr/bin/cp -R "$ARCHIE_TEST_SOURCE_DIR/scripts" "$source_dir/scripts"
|
|
for c in archied archie-gateway archie-state-store archie-ui archie-messaging archie-playbooks archie-agent; do
|
|
mkdir -p "$source_dir/cmd/$c"
|
|
done
|
|
: > "$source_dir/docker-compose.yml"
|
|
;;
|
|
*"rev-parse --verify refs/tags/"*)
|
|
tag="$(printf '%s' "$*" | sed 's/.*refs\/tags\///; s/\^.*//')"
|
|
echo "${tag//\//-}"
|
|
;;
|
|
esac
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "go", `
|
|
printf '%s\n' "go $*" >> "$ARCHIE_TEST_CALLS"
|
|
# The pre-flight task-database backup runs through the release's own
|
|
# archie-state-store subcommand, so a release whose recovery path cannot run
|
|
# must refuse the update.
|
|
if [ "$1" = "run" ]; then
|
|
[ "${ARCHIE_TEST_GO_FAILS_RUN:-}" = "1" ] && exit 1
|
|
# The PostgreSQL snapshot is the backup run that names no -db file.
|
|
[ "${ARCHIE_TEST_GO_FAILS_SNAPSHOT:-}" = "1" ] && [[ " $* " != *" -db "* ]] && exit 1
|
|
exit 0
|
|
fi
|
|
out=""
|
|
name=""
|
|
version="dev"
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-o" ]; then out="$2"; fi
|
|
case "$1" in
|
|
./cmd/*) name="${1#./cmd/}" ;;
|
|
*buildinfo.Version=*)
|
|
version="${1##*buildinfo.Version=}"
|
|
version="${version%% *}"
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
mkdir -p "$(dirname "$out")"
|
|
# The installed binary must answer -version the way a real one does, or the
|
|
# updater's skew check cannot read it.
|
|
printf '#!/usr/bin/env bash\necho "%s %s"\n' "$name" "$version" > "$out"
|
|
chmod 755 "$out"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "docker", `
|
|
printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$1" = image ] && [ "$2" = save ]; then
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = --output ]; then
|
|
: > "$2"
|
|
break
|
|
fi
|
|
shift
|
|
done
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "systemd-run", `printf '%s\n' "systemd-run $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "systemctl", `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
if [ "$2" = list-unit-files ]; then
|
|
# The reference deployment runs exactly these five as units. A command with
|
|
# no unit -- archie-playbooks, archie-agent -- has to look unitless, or the
|
|
# coverage guard would mistake every cmd/ directory for a host service.
|
|
present="${ARCHIE_TEST_PRESENT_UNITS:-archied.service archie-gateway.service archie-state-store.service archie-ui.service archie-messaging.service}"
|
|
case " ${ARCHIE_TEST_ABSENT_UNITS:-} " in
|
|
*" $3 "*) exit 0 ;;
|
|
esac
|
|
case " $present " in
|
|
*" $3 "*) printf '%s enabled enabled\n' "$3" ;;
|
|
esac
|
|
fi
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "install", `
|
|
printf '%s\n' "install $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/install "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "cp", `
|
|
printf '%s\n' "cp $*" >> "$ARCHIE_TEST_CALLS"
|
|
/usr/bin/cp "$@"
|
|
`)
|
|
writeFakeCommand(t, fakeDir, "sqlite3", `
|
|
printf '%s\n' "sqlite3 $*" >> "$ARCHIE_TEST_CALLS"
|
|
[ "${ARCHIE_TEST_SQLITE3_FAILS:-}" = "1" ] && exit 1
|
|
# Reproduce ".backup '<path>'" by writing a plausible backup beside the
|
|
# source, so the watchdog's existence check behaves as it would in production.
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
.backup*) target="$(printf '%s' "$arg" | sed "s/^\.backup[[:space:]]*//; s/^'//; s/'$//")" ;;
|
|
esac
|
|
done
|
|
if [ -n "${target:-}" ]; then
|
|
mkdir -p "$(dirname "$target")"
|
|
printf 'backup\n' > "$target"
|
|
fi
|
|
`)
|
|
|
|
// The production adapter prepends /usr/local/go/bin. Run an otherwise
|
|
// byte-for-byte copy with the harness PATH left first so fake go records
|
|
// build intent without compiling or touching Docker/systemd.
|
|
scriptData, err := os.ReadFile(filepath.Join(root, "scripts", "archie-update-install"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
script := string(scriptData)
|
|
if edit != nil {
|
|
script = edit(script)
|
|
}
|
|
scriptPath := filepath.Join(work, "archie-update-install")
|
|
script = strings.Replace(script, `export PATH="/usr/local/go/bin:$PATH"`, `export PATH="$PATH"`, 1)
|
|
if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.CommandContext(ctx, scriptPath)
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_SOURCE_URL=https://example.invalid/archie-core.git",
|
|
"ARCHIE_TEST_SOURCE_DIR="+root,
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_CONFIG_PATH="+configPath,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
)
|
|
for key, value := range environment {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
output, err := cmd.CombinedOutput()
|
|
switch {
|
|
case wantErr && err == nil:
|
|
t.Fatalf("archie-update-install succeeded, want refusal:\n%s", output)
|
|
case !wantErr && err != nil:
|
|
t.Fatalf("archie-update-install failed: %v\n%s", err, output)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
if wantErr {
|
|
return Result{}, calls, string(output)
|
|
}
|
|
var result Result
|
|
scanner := bufio.NewScanner(strings.NewReader(string(output)))
|
|
found := false
|
|
for scanner.Scan() {
|
|
if payload, ok := strings.CutPrefix(scanner.Text(), updateResultSentinel); ok {
|
|
found = true
|
|
if err := json.Unmarshal([]byte(payload), &result); err != nil {
|
|
t.Fatalf("decode result %q: %v", payload, err)
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("installer output has no result sentinel:\n%s", output)
|
|
}
|
|
return result, calls, string(output)
|
|
}
|
|
|
|
func writeFakeBinary(t *testing.T, dir, name, version string) {
|
|
t.Helper()
|
|
body := "#!/usr/bin/env bash\necho " + strconv.Quote(name+" "+version) + "\n"
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func writeFakeCommand(t *testing.T, dir, name, body string) {
|
|
t.Helper()
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := filepath.Join(dir, name)
|
|
if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\nset -euo pipefail\n"+body), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// watchdogRun configures one end-to-end run of the watchdog script. curl is
|
|
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
|
|
// which is what turns a health probe into a genuine test of the URL the
|
|
// watchdog builds rather than of a stub that ignores it.
|
|
type watchdogRun struct {
|
|
components string
|
|
curl string
|
|
// systemctl replaces the fake systemctl on PATH, so a case can observe or
|
|
// condition what the watchdog does between units. Empty keeps the inert
|
|
// `exit 0` stub.
|
|
systemctl string
|
|
env map[string]string
|
|
}
|
|
|
|
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
|
|
t.Helper()
|
|
return runUpdateWatchdog(t, watchdogRun{
|
|
components: components,
|
|
curl: `exit 1`,
|
|
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
|
|
})
|
|
}
|
|
|
|
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir := filepath.Join(work, "bin")
|
|
fakeDir := filepath.Join(work, "fake-bin")
|
|
reportPath := filepath.Join(work, "report.json")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
files := map[string]string{
|
|
"archied": "new daemon", "archied.prev": "old daemon",
|
|
"archie-agent-image.prev.tar": "saved old image",
|
|
}
|
|
for name, content := range files {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
systemctl := run.systemctl
|
|
if systemctl == "" {
|
|
systemctl = `exit 0`
|
|
}
|
|
writeFakeCommand(t, fakeDir, "systemctl", systemctl)
|
|
if run.curl != "" {
|
|
writeFakeCommand(t, fakeDir, "curl", run.curl)
|
|
}
|
|
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
callsPath := filepath.Join(work, "calls")
|
|
|
|
cmd := exec.CommandContext(ctx, filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(
|
|
os.Environ(),
|
|
"PATH="+fakeDir+":"+os.Getenv("PATH"),
|
|
"ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath,
|
|
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
|
|
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
|
|
"ARCHIE_UPDATE_COMPONENTS="+run.components,
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
|
|
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
|
|
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
|
|
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
|
|
)
|
|
for key, value := range run.env {
|
|
cmd.Env = append(cmd.Env, key+"="+value)
|
|
}
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
reportData, err := os.ReadFile(reportPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var report Report
|
|
if err := json.Unmarshal(reportData, &report); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
daemon, err := os.ReadFile(filepath.Join(binDir, "archied"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
callsData, err := os.ReadFile(callsPath)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
var calls []string
|
|
if len(callsData) > 0 {
|
|
calls = strings.Split(strings.TrimSpace(string(callsData)), "\n")
|
|
}
|
|
return report, string(daemon), calls
|
|
}
|
|
|
|
func assertCallContains(t *testing.T, calls []string, fragments ...string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
matched := true
|
|
for _, fragment := range fragments {
|
|
matched = matched && strings.Contains(call, fragment)
|
|
}
|
|
if matched {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("calls %#v contain no call with fragments %#v", calls, fragments)
|
|
}
|
|
|
|
func assertCallAbsent(t *testing.T, calls []string, fragment string) {
|
|
t.Helper()
|
|
for _, call := range calls {
|
|
if strings.Contains(call, fragment) {
|
|
t.Fatalf("calls unexpectedly contain %q: %#v", fragment, calls)
|
|
}
|
|
}
|
|
}
|
|
|
|
// An archied release is five processes. Installing the daemon binary alone
|
|
// left the State Store, Gateway and UI on the previous release, so archied
|
|
// came up against a State Store nothing had started, failed its health check
|
|
// and rolled back (archie-core-hbqk).
|
|
func TestUpdateInstallBuildsEveryReleaseBinary(t *testing.T) {
|
|
_, calls := runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
})
|
|
|
|
for _, cmd := range []string{"archied", "archie-gateway", "archie-state-store", "archie-ui", "archie-messaging", "archie-playbooks"} {
|
|
assertCallContains(t, calls, "go build", "internal/app/archied.gatewayVersion=1.23.0", "./cmd/"+cmd)
|
|
assertCallContains(t, calls, "install -m755", "/"+cmd)
|
|
}
|
|
assertCallAbsent(t, calls, "./cmd/archie-agent")
|
|
// The watchdog cannot restart what it is not told about.
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_UNITS=archie-state-store archie-gateway archied archie-ui archie-messaging")
|
|
assertCallContains(t, calls, "systemd-run",
|
|
"--setenv=ARCHIE_UPDATE_BINARIES=archie-state-store archie-gateway archied archie-ui archie-messaging archie-playbooks")
|
|
}
|
|
|
|
// Installing binaries for processes the host has no unit for produces exactly
|
|
// the failure this bug was: archied dials a State Store nothing starts. Refuse
|
|
// before touching the host instead, and say what is missing.
|
|
func TestUpdateInstallRefusesWhenAServiceUnitIsMissing(t *testing.T) {
|
|
_, calls, output := runUpdateInstall(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_ABSENT_UNITS": "archie-state-store.service",
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-state-store.service") {
|
|
t.Errorf("refusal must name the missing unit; output = %q", output)
|
|
}
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
// Nothing may be replaced on a host it would leave broken.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// The other half of the same contract, and the failure that actually happened
|
|
// (archie-core-1faq): the installed updater's GATEWAY_SERVICES predated
|
|
// archie-messaging, so the update neither backed it up nor restarted it and a
|
|
// hand-installed binary kept serving under an otherwise updated host. A unit
|
|
// this host runs must be named by the list, or the update refuses and says which.
|
|
func TestUpdateInstallRefusesWhenTheServiceListOmitsAHostUnit(t *testing.T) {
|
|
_, calls, output := runUpdateInstallEdited(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", func(script string) string {
|
|
// Drop archie-messaging from the list without touching its unit: the
|
|
// stale installed updater's view of the host.
|
|
return strings.Replace(script,
|
|
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui archie-messaging"`,
|
|
`GATEWAY_SERVICES="archie-state-store archie-gateway archied archie-ui"`,
|
|
1)
|
|
}, true)
|
|
|
|
if !strings.Contains(output, "archie-messaging.service") {
|
|
t.Errorf("refusal must name the unmanaged unit; output = %q", output)
|
|
}
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
// Nothing may be replaced on a host the update would leave partial.
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
}
|
|
|
|
// Carina's actual pair (archie-core-k94o): archie-messaging 1.35.0 beside an
|
|
// archie-state-store 1.30.0, the daemon at 1.30.0, and nothing approved to
|
|
// install. The component-driven updater reported only "nothing to install"
|
|
// while the two disagreed; now it reads each installed binary and names them.
|
|
func TestUpdateInstallRefusesWhenInstalledBinariesDisagree(t *testing.T) {
|
|
_, calls, output := runUpdateInstallFull(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
|
|
"archie-messaging": "1.35.0",
|
|
"archie-state-store": "1.30.0",
|
|
}, nil, true)
|
|
|
|
if !strings.Contains(output, "archie-messaging=1.35.0") {
|
|
t.Errorf("refusal must name the skewed binary and its version; output = %q", output)
|
|
}
|
|
if strings.Contains(output, "archie-state-store=") {
|
|
t.Errorf("refusal named a binary that agrees; output = %q", output)
|
|
}
|
|
assertCallAbsent(t, calls, "go build")
|
|
assertCallAbsent(t, calls, "install -m755")
|
|
t.Logf("refusal output:\n%s", strings.TrimSpace(output))
|
|
}
|
|
|
|
// A binary built outside a release reports "dev"; on a released host that is
|
|
// skew too, and it must be named rather than read as agreement.
|
|
func TestUpdateInstallRefusesAnUnstampedBinary(t *testing.T) {
|
|
_, _, output := runUpdateInstallFull(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.30.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.30.0",
|
|
}, "[containers]\nimage = 'registry.example/archie-agent:stable'\n", map[string]string{
|
|
"archie-ui": "dev",
|
|
}, nil, true)
|
|
|
|
if !strings.Contains(output, "archie-ui=dev") {
|
|
t.Errorf("refusal must name the unstamped binary; output = %q", output)
|
|
}
|
|
}
|
|
|
|
// The deployment is several processes, so the watchdog must cycle all of them,
|
|
// and in dependency order: the State Store owns archie.db and everything dials
|
|
// it, so it starts first and stops last (archie-core-hbqk).
|
|
func TestUpdateWatchdogCyclesEveryUnitInDependencyOrder(t *testing.T) {
|
|
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
work := t.TempDir()
|
|
binDir, fakeDir := filepath.Join(work, "bin"), filepath.Join(work, "fake-bin")
|
|
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// archie-ui is new in this release: it has no .prev, so rolling back must
|
|
// remove it rather than leave a binary from a release that was withdrawn.
|
|
for name, content := range map[string]string{
|
|
"archied": "new", "archied.prev": "old",
|
|
"archie-state-store": "new", "archie-state-store.prev": "old",
|
|
"archie-ui": "new",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
callsPath := filepath.Join(work, "calls")
|
|
writeFakeCommand(t, fakeDir, "systemctl", `printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"`)
|
|
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
|
cmd := exec.CommandContext(t.Context(), filepath.Join(root, "scripts", "archie-update-watchdog"))
|
|
cmd.Env = append(os.Environ(), "PATH="+fakeDir+":"+os.Getenv("PATH"), "ARCHIE_BIN_DIR="+binDir,
|
|
"ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
|
"ARCHIE_UPDATE_COMPONENTS=daemon",
|
|
"ARCHIE_UPDATE_UNITS=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_BINARIES=archie-state-store archied archie-ui",
|
|
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.22.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.23.0")
|
|
if output, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
|
}
|
|
calls, err := os.ReadFile(callsPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
indexOf := func(want string) int {
|
|
for i, line := range strings.Split(strings.TrimSpace(string(calls)), "\n") {
|
|
if line == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
startStore := indexOf("systemctl --user restart archie-state-store.service")
|
|
startDaemon := indexOf("systemctl --user restart archied.service")
|
|
stopUI := indexOf("systemctl --user stop archie-ui.service")
|
|
stopStore := indexOf("systemctl --user stop archie-state-store.service")
|
|
if startStore < 0 || startDaemon < 0 || startStore > startDaemon {
|
|
t.Errorf("State Store must start before archied; calls =\n%s", calls)
|
|
}
|
|
if stopUI < 0 || stopStore < 0 || stopUI > stopStore {
|
|
t.Errorf("UI must stop before the State Store; calls =\n%s", calls)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(binDir, "archie-ui")); !os.IsNotExist(err) {
|
|
t.Errorf("archie-ui had no .prev and must be removed on rollback, stat err = %v", err)
|
|
}
|
|
for _, name := range []string{"archied", "archie-state-store"} {
|
|
got, err := os.ReadFile(filepath.Join(binDir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(got) != "old" {
|
|
t.Errorf("%s = %q after rollback, want the previous binary", name, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The updater must back up the database the daemon actually uses, which is
|
|
// `<db_path>-tasks.sqlite` (taskDBPath in internal/app/archied/main.go), not
|
|
// the configured path itself. The configured path may be a zero-byte
|
|
// placeholder while the real store sits beside it -- so backing the literal
|
|
// value up produces an empty backup, and a rollback then restores that empty
|
|
// file over the database and deletes its -wal/-shm. A rollback that destroys
|
|
// the task store is worse than no rollback.
|
|
//
|
|
// The snapshot is taken by the release's own recovery subcommand rather than
|
|
// by hand-rolled shell, so the installer and an operator run the same code and
|
|
// the two cannot drift apart.
|
|
func TestUpdateInstallBacksUpTheSiblingTaskDatabase(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
realStore := configured + "-tasks.sqlite"
|
|
if err := os.WriteFile(configured, nil, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(realStore, []byte("real task data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
backup := filepath.Join(work, "backup.sqlite")
|
|
|
|
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TASK_DB_BACKUP": backup,
|
|
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
|
|
|
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore, "-out "+backup)
|
|
assertCallAbsent(t, calls, "sqlite3")
|
|
assertCallContains(t, calls, "systemd-run", "--setenv=ARCHIE_TASK_DB_PATH="+realStore)
|
|
// The zero-byte placeholder is not the store, and must never be the thing
|
|
// handed to the watchdog as ARCHIE_TASK_DB_PATH.
|
|
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+configured+" ")
|
|
}
|
|
|
|
// db_path can also appear under a TOML table -- the retired [indexing] section
|
|
// once owned its own database there, and a future section could do the same.
|
|
// Reading the first match in the file can select an unrelated database, so the
|
|
// value must come from the daemon's own top-level key.
|
|
func TestUpdateInstallIgnoresDbPathFromOtherSections(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
realStore := configured + "-tasks.sqlite"
|
|
indexStore := filepath.Join(work, "workspace-indexes.db")
|
|
for path, content := range map[string]string{realStore: "real task data", indexStore: "index data"} {
|
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
backup := filepath.Join(work, "backup.sqlite")
|
|
|
|
_, calls, _ := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TASK_DB_BACKUP": backup,
|
|
}, "db_path = \""+configured+"\"\n\n[indexing]\ndb_path = \""+indexStore+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", false)
|
|
|
|
assertCallContains(t, calls, "go run ./cmd/archie-state-store backup", "-db "+realStore)
|
|
for _, call := range calls {
|
|
if strings.Contains(call, "go run ./cmd/archie-state-store backup") && strings.Contains(call, indexStore) {
|
|
t.Errorf("backed up the other section's database %s: %q", indexStore, call)
|
|
}
|
|
}
|
|
assertCallAbsent(t, calls, "--setenv=ARCHIE_TASK_DB_PATH="+indexStore)
|
|
}
|
|
|
|
// A store that cannot be backed up must fail the update before anything is
|
|
// replaced. Installing with no usable rollback is how a failed update becomes
|
|
// a destroyed task store.
|
|
func TestUpdateInstallRefusesWhenTaskDatabaseCannotBeBackedUp(t *testing.T) {
|
|
work := t.TempDir()
|
|
configured := filepath.Join(work, "archie.db")
|
|
if err := os.WriteFile(configured+"-tasks.sqlite", []byte("data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, calls, output := runUpdateInstallWithConfig(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.22.0",
|
|
"ARCHIE_UPDATE_DAEMON_VERSION": "1.23.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.21.0",
|
|
"ARCHIE_TEST_GO_FAILS_RUN": "1",
|
|
}, "db_path = \""+configured+"\"\n\n[containers]\nimage = 'registry.example/archie-agent:stable'\n", true)
|
|
|
|
if !strings.Contains(output, "task database") {
|
|
t.Errorf("refusal must name the task database; output = %q", output)
|
|
}
|
|
// Nothing of the release may be placed on the host, and no backup of the
|
|
// failure path may be attempted either.
|
|
for _, call := range calls {
|
|
if strings.Contains(call, "install -m755") && !strings.Contains(call, "/scripts/") {
|
|
t.Errorf("refused update still installed %q", call)
|
|
}
|
|
}
|
|
assertCallAbsent(t, calls, "arhied.prev")
|
|
}
|
|
|
|
// The sidecar is the only record of the managed agent version, and
|
|
// archie-update-check refuses anything but a bare release version, so a stray
|
|
// escape in the write leaves the installed agent reported as unknown forever.
|
|
func TestUpdateInstallWritesABareAgentVersionSidecar(t *testing.T) {
|
|
versionFile := filepath.Join(t.TempDir(), "archie-agent.version")
|
|
runUpdateInstallScript(t, map[string]string{
|
|
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.13.0",
|
|
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
|
"ARCHIE_UPDATE_AGENT_VERSION": "1.10.0",
|
|
"ARCHIE_AGENT_VERSION_FILE": versionFile,
|
|
})
|
|
|
|
recorded, err := os.ReadFile(versionFile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(recorded) != "1.10.0\n" {
|
|
t.Fatalf("sidecar = %q, want %q", recorded, "1.10.0\n")
|
|
}
|
|
}
|
|
|
|
// freeAddress reserves a loopback address nothing is listening on and releases
|
|
// it, so a case can hand the watchdog a State Store target whose port stays
|
|
// closed until the fake State Store binds it.
|
|
func freeAddress(t *testing.T) string {
|
|
t.Helper()
|
|
var listen net.ListenConfig
|
|
listener, err := listen.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
address := listener.Addr().String()
|
|
if err := listener.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return address
|
|
}
|
|
|
|
// TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied pins why an
|
|
// ordered start is not enough on its own.
|
|
//
|
|
// `systemctl --user start` returns once the process is forked, not once its port
|
|
// accepts. On a real 1.37.1 -> 1.38.0 update the State Store's unit reported
|
|
// started while it was still binding, archied started next, dialled a closed
|
|
// port, exited 1, and was rescued only by systemd's Restart=on-failure seven
|
|
// seconds later -- behind a "build and install succeeded" line, on a unit that
|
|
// need not carry that restart policy at all.
|
|
//
|
|
// The fake State Store binds its port two seconds AFTER systemctl returns, which
|
|
// is the race. The fake systemctl for archied records whether the port was open
|
|
// at the moment it was asked to start, so a watchdog that starts archied without
|
|
// waiting says so in the call log rather than passing quietly.
|
|
func TestUpdateWatchdogWaitsForTheStateStoreBeforeStartingArchied(t *testing.T) {
|
|
target := freeAddress(t)
|
|
_, _, calls := runUpdateWatchdog(t, watchdogRun{
|
|
components: "daemon",
|
|
curl: `exit 1`,
|
|
systemctl: `
|
|
printf '%s\n' "systemctl $*" >> "$ARCHIE_TEST_CALLS"
|
|
case "$*" in
|
|
*"restart archie-state-store.service")
|
|
# Bind two seconds late, then hold the port: the State Store's unit returning
|
|
# before its listener exists is the whole failure being guarded against.
|
|
( sleep 2; exec python3 -c "import os,socket,time; h,p=os.environ['ARCHIE_UPDATE_STATE_STORE_TARGET'].rsplit(':',1); s=socket.socket(); s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1); s.bind((h,int(p))); s.listen(8); time.sleep(30)" ) >/dev/null 2>&1 &
|
|
;;
|
|
*"restart archied.service")
|
|
if ! (exec 3<>"/dev/tcp/${ARCHIE_UPDATE_STATE_STORE_TARGET%:*}/${ARCHIE_UPDATE_STATE_STORE_TARGET##*:}") 2>/dev/null; then
|
|
printf '%s\n' 'archied started before the state store accepted' >> "$ARCHIE_TEST_CALLS"
|
|
fi
|
|
;;
|
|
esac
|
|
exit 0`,
|
|
env: map[string]string{
|
|
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0",
|
|
"ARCHIE_UPDATE_UNITS": "archie-state-store archie-gateway archied",
|
|
"ARCHIE_UPDATE_STATE_STORE_TARGET": target,
|
|
"ARCHIE_UPDATE_STATE_STORE_TIMEOUT": "15",
|
|
},
|
|
})
|
|
|
|
assertCallContains(t, calls, "restart archie-state-store.service")
|
|
assertCallContains(t, calls, "restart archie-gateway.service")
|
|
assertCallContains(t, calls, "restart archied.service")
|
|
assertCallAbsent(t, calls, "archied started before the state store accepted")
|
|
}
|