diff --git a/internal/app/archied/telegram_setup.go b/internal/app/archied/telegram_setup.go index f88953ce5..d0508987d 100644 --- a/internal/app/archied/telegram_setup.go +++ b/internal/app/archied/telegram_setup.go @@ -26,6 +26,7 @@ import ( "github.com/samcharles93/archie-core/internal/secret" "github.com/samcharles93/archie-core/internal/store" "github.com/samcharles93/archie-core/internal/tools" + "github.com/samcharles93/archie-core/internal/webui" ) // telegramSetup contains the inputs needed to initialise the Telegram @@ -235,7 +236,10 @@ func makeUpdateService(s telegramSetup) *releaseupdate.Service { Enrich: componentInstallTypeEnricher(s.AgentStatus, cfg.NATS), } if len(cfg.Chat.Telegram.UpdateInstallCommand) != 0 { - updates.Installer = releaseupdate.CommandInstaller{Command: cfg.Chat.Telegram.UpdateInstallCommand} + updates.Installer = releaseupdate.CommandInstaller{ + Command: cfg.Chat.Telegram.UpdateInstallCommand, + HealthURL: webui.HealthURL(cfg.Web.Listen), + } } return updates } diff --git a/internal/app/archied/telegram_setup_test.go b/internal/app/archied/telegram_setup_test.go index b0cbe36d3..3d7493914 100644 --- a/internal/app/archied/telegram_setup_test.go +++ b/internal/app/archied/telegram_setup_test.go @@ -200,3 +200,36 @@ func TestDaemonRunningVersionsIncludesObservedAgent(t *testing.T) { t.Errorf("running[%q] = %q, want the observed version %q", releaseupdate.ComponentAgent, got, "1.9.11") } } + +// TestMakeUpdateServiceDerivesHealthURLFromWebListen: the watchdog's health +// probe has to reach the dashboard listener this instance actually binds. +// Left to the script's own default it polls a port nothing serves, waits out +// the timeout and rolls back a release that came up fine (archie-core-1r4g). +func TestMakeUpdateServiceDerivesHealthURLFromWebListen(t *testing.T) { + for _, tc := range []struct{ name, listen, want string }{ + {name: "loopback dashboard", listen: "127.0.0.1:9000", want: "http://127.0.0.1:9000"}, + {name: "wildcard bind", listen: "0.0.0.0:8484", want: "http://localhost:8484"}, + {name: "dashboard off", listen: "off", want: ""}, + {name: "dashboard unset", listen: "", want: ""}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config.Config{} + cfg.Web.Listen = tc.listen + cfg.Chat.Telegram.UpdateCheckCommand = []string{"archie-update-check"} + cfg.Chat.Telegram.UpdateInstallCommand = []string{"archie-update-install"} + + service := makeUpdateService(telegramSetup{Cfg: config.NewHolder(cfg)}) + + if service == nil { + t.Fatal("makeUpdateService returned nil despite a configured check command") + } + installer, ok := service.Installer.(releaseupdate.CommandInstaller) + if !ok { + t.Fatalf("Service.Installer = %T, want releaseupdate.CommandInstaller", service.Installer) + } + if installer.HealthURL != tc.want { + t.Errorf("Installer.HealthURL = %q, want %q", installer.HealthURL, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/command.go b/internal/releaseupdate/command.go index 55d43e643..b1f6438ad 100644 --- a/internal/releaseupdate/command.go +++ b/internal/releaseupdate/command.go @@ -36,7 +36,15 @@ func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) { return snapshot, nil } -type CommandInstaller struct{ Command []string } +// CommandInstaller runs the deployment's own install command. HealthURL is +// the base URL its restart tooling must poll for /healthz -- only this +// process knows which address it actually serves, so a script left to guess +// one rolls back releases that came up healthy (archie-core-1r4g). Empty +// leaves ARCHIE_HEALTH_URL as the environment already has it. +type CommandInstaller struct { + Command []string + HealthURL string +} // Install runs the configured command and streams its stdout to progress // line by line as the command emits it -- the earlier CombinedOutput-based @@ -60,6 +68,9 @@ func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta I "ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed, "ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available, ) + if i.HealthURL != "" { + cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL) + } stdout, err := cmd.StdoutPipe() if err != nil { diff --git a/internal/releaseupdate/command_test.go b/internal/releaseupdate/command_test.go index 106fb6279..1c4f2c9ae 100644 --- a/internal/releaseupdate/command_test.go +++ b/internal/releaseupdate/command_test.go @@ -151,3 +151,32 @@ func TestCommandInstallerEmptyCommandReturnsError(t *testing.T) { t.Fatal("Install() error = nil, want empty-command error") } } + +// TestCommandInstallerForwardsHealthURL: only the daemon knows which address +// it serves /healthz on, so it, not the install script, decides what the +// post-restart probe polls (archie-core-1r4g). +func TestCommandInstallerForwardsHealthURL(t *testing.T) { + script := writeFixtureScript(t, ` +echo "health=${ARCHIE_HEALTH_URL:-unset}" +`) + t.Setenv("ARCHIE_HEALTH_URL", "http://operator.example:9999") + + for _, tc := range []struct{ name, healthURL, want string }{ + {name: "configured", healthURL: "http://127.0.0.1:9000", want: "health=http://127.0.0.1:9000"}, + // An unset field must not clobber an operator's own export: a + // deployment with the dashboard off has no address to derive and + // the exported value is the only one there is. + {name: "unset leaves the environment alone", want: "health=http://operator.example:9999"}, + } { + t.Run(tc.name, func(t *testing.T) { + installer := CommandInstaller{Command: []string{script}, HealthURL: tc.healthURL} + var progress []string + if _, err := installer.Install(context.Background(), Snapshot{}, InstallMeta{}, func(line string) { progress = append(progress, line) }); err != nil { + t.Fatalf("Install() error = %v", err) + } + if len(progress) != 1 || progress[0] != tc.want { + t.Fatalf("progress = %#v, want [%q]", progress, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index 73201d4cb..cb42fabf5 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -4,6 +4,8 @@ import ( "bufio" "database/sql" "encoding/json" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -90,6 +92,21 @@ func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) { } } +// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between +// the two: systemd-run starts the watchdog with a clean environment, so a +// health address the daemon derived reaches the probe only if the install +// script hands it over explicitly (archie-core-1r4g). +func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) { + _, calls := runUpdateInstallScript(t, map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0", + "ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9", + "ARCHIE_HEALTH_URL": "http://127.0.0.1:9000", + }) + + assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000") +} + func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) { tests := []struct { name string @@ -125,6 +142,51 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent } } +// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch +// no other test reaches: a daemon that comes back up. Every other watchdog +// test fakes curl as an immediate failure, so before this one the probe URL +// the script actually builds was never dialled and its default could point +// anywhere (archie-core-1r4g). +func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + daemon := httptest.NewServer(mux) + defer daemon.Close() + + report, installed, calls := runUpdateWatchdog(t, watchdogRun{ + components: "daemon", + env: map[string]string{ + "ARCHIE_HEALTH_URL": daemon.URL, + "ARCHIE_UPDATE_HEALTH_TIMEOUT": "10", + }, + }) + + if report.HealthCheck != "passed" || report.RolledBack { + t.Fatalf("report = %#v, want a passed health check with no rollback", report) + } + if installed != "new daemon" { + t.Fatalf("installed daemon = %q, want the new binary left in place", installed) + } + assertCallAbsent(t, calls, "docker image load") +} + +// TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener pins the +// fallback the script uses when nothing exports one. It has to be the +// address archied serves /healthz on (config.example.toml's [web] listen); +// anything else times out and rolls back a healthy release. +func TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener(t *testing.T) { + report, _, calls := runUpdateWatchdog(t, watchdogRun{ + components: "daemon", + curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`, + env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"}, + }) + + assertCallContains(t, calls, "curl ", "http://127.0.0.1:8484/healthz") + if report.HealthCheck != "passed" || report.RolledBack { + t.Fatalf("report = %#v, want a passed health check with no rollback", report) + } +} + func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) { root, err := filepath.Abs(filepath.Join("..", "..")) if err != nil { @@ -343,7 +405,26 @@ func writeFakeCommand(t *testing.T, dir, name, body string) { } } +// watchdogRun configures one end-to-end run of the watchdog script. curl is +// the fake curl placed on PATH; leaving it empty keeps the real curl there, +// which is what turns a health probe into a genuine test of the URL the +// watchdog builds rather than of a stub that ignores it. +type watchdogRun struct { + components string + curl string + env map[string]string +} + func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) { + t.Helper() + return runUpdateWatchdog(t, watchdogRun{ + components: components, + curl: `exit 1`, + env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"}, + }) +} + +func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) { t.Helper() ctx := t.Context() @@ -368,7 +449,9 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, } } writeFakeCommand(t, fakeDir, "systemctl", `exit 0`) - writeFakeCommand(t, fakeDir, "curl", `exit 1`) + if run.curl != "" { + writeFakeCommand(t, fakeDir, "curl", run.curl) + } writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`) callsPath := filepath.Join(work, "calls") @@ -380,13 +463,15 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, "ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable", "ARCHIE_UPDATE_REPORT_PATH="+reportPath, - "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", - "ARCHIE_UPDATE_COMPONENTS="+components, + "ARCHIE_UPDATE_COMPONENTS="+run.components, "ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0", "ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9", "ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0", ) + for key, value := range run.env { + cmd.Env = append(cmd.Env, key+"="+value) + } if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("watchdog failed: %v\n%s", err, output) } diff --git a/internal/webui/auth.go b/internal/webui/auth.go index ab738a29a..739392976 100644 --- a/internal/webui/auth.go +++ b/internal/webui/auth.go @@ -60,6 +60,24 @@ func DashboardURL(listen, token string) string { return url } +// HealthURL renders the base URL of this process's /healthz endpoint for +// out-of-process tooling: the update watchdog polls it after restarting +// archied to decide whether the release it just installed came up. +// +// It is empty when the dashboard is disabled, because /healthz is served by +// the dashboard listener and there is no other surface to probe. +// +// The address must remain the one archied itself binds. The watchdog is +// verifying archied's own restart, so probing an address served by a +// different process would report health for something it never restarted. +func HealthURL(listen string) string { + listen = strings.TrimSpace(listen) + if listen == "" || listen == "off" { + return "" + } + return strings.TrimSuffix(DashboardURL(listen, ""), "/") +} + // LoadOrCreateToken returns the dashboard token, generating and persisting one // on first use. The token is created rather than configured so a non-loopback // bind cannot be brought up unprotected by omission: there is no setup step to diff --git a/internal/webui/dashboard_url_test.go b/internal/webui/dashboard_url_test.go index 71d9bfbde..ff251bdfe 100644 --- a/internal/webui/dashboard_url_test.go +++ b/internal/webui/dashboard_url_test.go @@ -15,3 +15,24 @@ func TestDashboardURL(t *testing.T) { } } } + +// TestHealthURL pins what out-of-process tooling probes after restarting +// archied. A wrong address here reads as an unhealthy release and triggers a +// rollback of one that came up fine (archie-core-1r4g). +func TestHealthURL(t *testing.T) { + for _, tc := range []struct{ listen, want string }{ + {"127.0.0.1:8484", "http://127.0.0.1:8484"}, + {"localhost:8484", "http://localhost:8484"}, + {"0.0.0.0:8484", "http://localhost:8484"}, + {":8484", "http://localhost:8484"}, + {"[::]:8484", "http://localhost:8484"}, + {"100.64.1.2:9000", "http://100.64.1.2:9000"}, + {" 127.0.0.1:8484 ", "http://127.0.0.1:8484"}, + {"off", ""}, + {"", ""}, + } { + if got := HealthURL(tc.listen); got != tc.want { + t.Errorf("HealthURL(%q) = %q, want %q", tc.listen, got, tc.want) + } + } +} diff --git a/scripts/archie-update-install b/scripts/archie-update-install index 9722d950b..2fb0bc620 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -166,7 +166,7 @@ fi echo "==> handing off to the update watchdog for restart and health verification" systemd-run --user --unit=archied-update-restart --collect --quiet \ --setenv=ARCHIE_BIN_DIR="$BIN_DIR" \ - --setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" \ + --setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \ --setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \ --setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \ --setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \ diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index 1a168b0f7..4ef9d4824 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -12,7 +12,11 @@ AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" -HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" +# The daemon exports its own dashboard address (releaseupdate.CommandInstaller); +# this default only covers a hand-driven run, and must stay the address archied +# serves /healthz on -- config.example.toml's [web] listen. +HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8484}" +HEALTH_URL="${HEALTH_URL%/}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" POLL_INTERVAL_SECONDS=3