From 2fe708aa6482ffa5239a48570dc0a4bf43de00fe Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 9 Sep 2026 22:42:50 +1000 Subject: [PATCH] fix(update): probe the dashboard's real address after a self-update The watchdog polled http://127.0.0.1:8420/healthz. Nothing binds 8420: the daemon serves /healthz on [web].listen (127.0.0.1:8484 by default), so every dashboard- or Telegram-initiated update waited out the 60s health timeout and rolled back a release that had come up fine. The daemon is the only process that knows which address it binds, so it now derives the probe URL from its own config and hands it to the installer; CommandInstaller exports ARCHIE_HEALTH_URL and the install script forwards it into the watchdog unit, whose remaining default matches the dashboard's. The suite never dialled the URL: every watchdog test faked curl as an immediate failure, so only the unhealthy branch existed. One test now runs the real curl against a real listener, and another pins the script default. --- internal/app/archied/telegram_setup.go | 6 +- internal/app/archied/telegram_setup_test.go | 33 +++++++ internal/releaseupdate/command.go | 13 ++- internal/releaseupdate/command_test.go | 29 ++++++ internal/releaseupdate/install_script_test.go | 91 ++++++++++++++++++- internal/webui/auth.go | 18 ++++ internal/webui/dashboard_url_test.go | 21 +++++ scripts/archie-update-install | 2 +- scripts/archie-update-watchdog | 6 +- 9 files changed, 212 insertions(+), 7 deletions(-) diff --git a/internal/app/archied/telegram_setup.go b/internal/app/archied/telegram_setup.go index f88953ce5..d0508987d 100644 --- a/internal/app/archied/telegram_setup.go +++ b/internal/app/archied/telegram_setup.go @@ -26,6 +26,7 @@ import ( "github.com/samcharles93/archie-core/internal/secret" "github.com/samcharles93/archie-core/internal/store" "github.com/samcharles93/archie-core/internal/tools" + "github.com/samcharles93/archie-core/internal/webui" ) // telegramSetup contains the inputs needed to initialise the Telegram @@ -235,7 +236,10 @@ func makeUpdateService(s telegramSetup) *releaseupdate.Service { Enrich: componentInstallTypeEnricher(s.AgentStatus, cfg.NATS), } if len(cfg.Chat.Telegram.UpdateInstallCommand) != 0 { - updates.Installer = releaseupdate.CommandInstaller{Command: cfg.Chat.Telegram.UpdateInstallCommand} + updates.Installer = releaseupdate.CommandInstaller{ + Command: cfg.Chat.Telegram.UpdateInstallCommand, + HealthURL: webui.HealthURL(cfg.Web.Listen), + } } return updates } diff --git a/internal/app/archied/telegram_setup_test.go b/internal/app/archied/telegram_setup_test.go index b0cbe36d3..3d7493914 100644 --- a/internal/app/archied/telegram_setup_test.go +++ b/internal/app/archied/telegram_setup_test.go @@ -200,3 +200,36 @@ func TestDaemonRunningVersionsIncludesObservedAgent(t *testing.T) { t.Errorf("running[%q] = %q, want the observed version %q", releaseupdate.ComponentAgent, got, "1.9.11") } } + +// TestMakeUpdateServiceDerivesHealthURLFromWebListen: the watchdog's health +// probe has to reach the dashboard listener this instance actually binds. +// Left to the script's own default it polls a port nothing serves, waits out +// the timeout and rolls back a release that came up fine (archie-core-1r4g). +func TestMakeUpdateServiceDerivesHealthURLFromWebListen(t *testing.T) { + for _, tc := range []struct{ name, listen, want string }{ + {name: "loopback dashboard", listen: "127.0.0.1:9000", want: "http://127.0.0.1:9000"}, + {name: "wildcard bind", listen: "0.0.0.0:8484", want: "http://localhost:8484"}, + {name: "dashboard off", listen: "off", want: ""}, + {name: "dashboard unset", listen: "", want: ""}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config.Config{} + cfg.Web.Listen = tc.listen + cfg.Chat.Telegram.UpdateCheckCommand = []string{"archie-update-check"} + cfg.Chat.Telegram.UpdateInstallCommand = []string{"archie-update-install"} + + service := makeUpdateService(telegramSetup{Cfg: config.NewHolder(cfg)}) + + if service == nil { + t.Fatal("makeUpdateService returned nil despite a configured check command") + } + installer, ok := service.Installer.(releaseupdate.CommandInstaller) + if !ok { + t.Fatalf("Service.Installer = %T, want releaseupdate.CommandInstaller", service.Installer) + } + if installer.HealthURL != tc.want { + t.Errorf("Installer.HealthURL = %q, want %q", installer.HealthURL, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/command.go b/internal/releaseupdate/command.go index 55d43e643..b1f6438ad 100644 --- a/internal/releaseupdate/command.go +++ b/internal/releaseupdate/command.go @@ -36,7 +36,15 @@ func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) { return snapshot, nil } -type CommandInstaller struct{ Command []string } +// CommandInstaller runs the deployment's own install command. HealthURL is +// the base URL its restart tooling must poll for /healthz -- only this +// process knows which address it actually serves, so a script left to guess +// one rolls back releases that came up healthy (archie-core-1r4g). Empty +// leaves ARCHIE_HEALTH_URL as the environment already has it. +type CommandInstaller struct { + Command []string + HealthURL string +} // Install runs the configured command and streams its stdout to progress // line by line as the command emits it -- the earlier CombinedOutput-based @@ -60,6 +68,9 @@ func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta I "ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed, "ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available, ) + if i.HealthURL != "" { + cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL) + } stdout, err := cmd.StdoutPipe() if err != nil { diff --git a/internal/releaseupdate/command_test.go b/internal/releaseupdate/command_test.go index 106fb6279..1c4f2c9ae 100644 --- a/internal/releaseupdate/command_test.go +++ b/internal/releaseupdate/command_test.go @@ -151,3 +151,32 @@ func TestCommandInstallerEmptyCommandReturnsError(t *testing.T) { t.Fatal("Install() error = nil, want empty-command error") } } + +// TestCommandInstallerForwardsHealthURL: only the daemon knows which address +// it serves /healthz on, so it, not the install script, decides what the +// post-restart probe polls (archie-core-1r4g). +func TestCommandInstallerForwardsHealthURL(t *testing.T) { + script := writeFixtureScript(t, ` +echo "health=${ARCHIE_HEALTH_URL:-unset}" +`) + t.Setenv("ARCHIE_HEALTH_URL", "http://operator.example:9999") + + for _, tc := range []struct{ name, healthURL, want string }{ + {name: "configured", healthURL: "http://127.0.0.1:9000", want: "health=http://127.0.0.1:9000"}, + // An unset field must not clobber an operator's own export: a + // deployment with the dashboard off has no address to derive and + // the exported value is the only one there is. + {name: "unset leaves the environment alone", want: "health=http://operator.example:9999"}, + } { + t.Run(tc.name, func(t *testing.T) { + installer := CommandInstaller{Command: []string{script}, HealthURL: tc.healthURL} + var progress []string + if _, err := installer.Install(context.Background(), Snapshot{}, InstallMeta{}, func(line string) { progress = append(progress, line) }); err != nil { + t.Fatalf("Install() error = %v", err) + } + if len(progress) != 1 || progress[0] != tc.want { + t.Fatalf("progress = %#v, want [%q]", progress, tc.want) + } + }) + } +} diff --git a/internal/releaseupdate/install_script_test.go b/internal/releaseupdate/install_script_test.go index 73201d4cb..cb42fabf5 100644 --- a/internal/releaseupdate/install_script_test.go +++ b/internal/releaseupdate/install_script_test.go @@ -4,6 +4,8 @@ import ( "bufio" "database/sql" "encoding/json" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -90,6 +92,21 @@ func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) { } } +// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between +// the two: systemd-run starts the watchdog with a clean environment, so a +// health address the daemon derived reaches the probe only if the install +// script hands it over explicitly (archie-core-1r4g). +func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) { + _, calls := runUpdateInstallScript(t, map[string]string{ + "ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0", + "ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0", + "ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9", + "ARCHIE_HEALTH_URL": "http://127.0.0.1:9000", + }) + + assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000") +} + func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) { tests := []struct { name string @@ -125,6 +142,51 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent } } +// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch +// no other test reaches: a daemon that comes back up. Every other watchdog +// test fakes curl as an immediate failure, so before this one the probe URL +// the script actually builds was never dialled and its default could point +// anywhere (archie-core-1r4g). +func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + daemon := httptest.NewServer(mux) + defer daemon.Close() + + report, installed, calls := runUpdateWatchdog(t, watchdogRun{ + components: "daemon", + env: map[string]string{ + "ARCHIE_HEALTH_URL": daemon.URL, + "ARCHIE_UPDATE_HEALTH_TIMEOUT": "10", + }, + }) + + if report.HealthCheck != "passed" || report.RolledBack { + t.Fatalf("report = %#v, want a passed health check with no rollback", report) + } + if installed != "new daemon" { + t.Fatalf("installed daemon = %q, want the new binary left in place", installed) + } + assertCallAbsent(t, calls, "docker image load") +} + +// TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener pins the +// fallback the script uses when nothing exports one. It has to be the +// address archied serves /healthz on (config.example.toml's [web] listen); +// anything else times out and rolls back a healthy release. +func TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener(t *testing.T) { + report, _, calls := runUpdateWatchdog(t, watchdogRun{ + components: "daemon", + curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`, + env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"}, + }) + + assertCallContains(t, calls, "curl ", "http://127.0.0.1:8484/healthz") + if report.HealthCheck != "passed" || report.RolledBack { + t.Fatalf("report = %#v, want a passed health check with no rollback", report) + } +} + func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) { root, err := filepath.Abs(filepath.Join("..", "..")) if err != nil { @@ -343,7 +405,26 @@ func writeFakeCommand(t *testing.T, dir, name, body string) { } } +// watchdogRun configures one end-to-end run of the watchdog script. curl is +// the fake curl placed on PATH; leaving it empty keeps the real curl there, +// which is what turns a health probe into a genuine test of the URL the +// watchdog builds rather than of a stub that ignores it. +type watchdogRun struct { + components string + curl string + env map[string]string +} + func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) { + t.Helper() + return runUpdateWatchdog(t, watchdogRun{ + components: components, + curl: `exit 1`, + env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"}, + }) +} + +func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) { t.Helper() ctx := t.Context() @@ -368,7 +449,9 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, } } writeFakeCommand(t, fakeDir, "systemctl", `exit 0`) - writeFakeCommand(t, fakeDir, "curl", `exit 1`) + if run.curl != "" { + writeFakeCommand(t, fakeDir, "curl", run.curl) + } writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`) callsPath := filepath.Join(work, "calls") @@ -380,13 +463,15 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, "ARCHIE_TEST_CALLS="+callsPath, "ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable", "ARCHIE_UPDATE_REPORT_PATH="+reportPath, - "ARCHIE_UPDATE_HEALTH_TIMEOUT=0", - "ARCHIE_UPDATE_COMPONENTS="+components, + "ARCHIE_UPDATE_COMPONENTS="+run.components, "ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0", "ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0", "ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9", "ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0", ) + for key, value := range run.env { + cmd.Env = append(cmd.Env, key+"="+value) + } if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("watchdog failed: %v\n%s", err, output) } diff --git a/internal/webui/auth.go b/internal/webui/auth.go index ab738a29a..739392976 100644 --- a/internal/webui/auth.go +++ b/internal/webui/auth.go @@ -60,6 +60,24 @@ func DashboardURL(listen, token string) string { return url } +// HealthURL renders the base URL of this process's /healthz endpoint for +// out-of-process tooling: the update watchdog polls it after restarting +// archied to decide whether the release it just installed came up. +// +// It is empty when the dashboard is disabled, because /healthz is served by +// the dashboard listener and there is no other surface to probe. +// +// The address must remain the one archied itself binds. The watchdog is +// verifying archied's own restart, so probing an address served by a +// different process would report health for something it never restarted. +func HealthURL(listen string) string { + listen = strings.TrimSpace(listen) + if listen == "" || listen == "off" { + return "" + } + return strings.TrimSuffix(DashboardURL(listen, ""), "/") +} + // LoadOrCreateToken returns the dashboard token, generating and persisting one // on first use. The token is created rather than configured so a non-loopback // bind cannot be brought up unprotected by omission: there is no setup step to diff --git a/internal/webui/dashboard_url_test.go b/internal/webui/dashboard_url_test.go index 71d9bfbde..ff251bdfe 100644 --- a/internal/webui/dashboard_url_test.go +++ b/internal/webui/dashboard_url_test.go @@ -15,3 +15,24 @@ func TestDashboardURL(t *testing.T) { } } } + +// TestHealthURL pins what out-of-process tooling probes after restarting +// archied. A wrong address here reads as an unhealthy release and triggers a +// rollback of one that came up fine (archie-core-1r4g). +func TestHealthURL(t *testing.T) { + for _, tc := range []struct{ listen, want string }{ + {"127.0.0.1:8484", "http://127.0.0.1:8484"}, + {"localhost:8484", "http://localhost:8484"}, + {"0.0.0.0:8484", "http://localhost:8484"}, + {":8484", "http://localhost:8484"}, + {"[::]:8484", "http://localhost:8484"}, + {"100.64.1.2:9000", "http://100.64.1.2:9000"}, + {" 127.0.0.1:8484 ", "http://127.0.0.1:8484"}, + {"off", ""}, + {"", ""}, + } { + if got := HealthURL(tc.listen); got != tc.want { + t.Errorf("HealthURL(%q) = %q, want %q", tc.listen, got, tc.want) + } + } +} diff --git a/scripts/archie-update-install b/scripts/archie-update-install index 9722d950b..2fb0bc620 100755 --- a/scripts/archie-update-install +++ b/scripts/archie-update-install @@ -166,7 +166,7 @@ fi echo "==> handing off to the update watchdog for restart and health verification" systemd-run --user --unit=archied-update-restart --collect --quiet \ --setenv=ARCHIE_BIN_DIR="$BIN_DIR" \ - --setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" \ + --setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \ --setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \ --setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \ --setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \ diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog index 1a168b0f7..4ef9d4824 100755 --- a/scripts/archie-update-watchdog +++ b/scripts/archie-update-watchdog @@ -12,7 +12,11 @@ AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}" AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}" TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}" TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}" -HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" +# The daemon exports its own dashboard address (releaseupdate.CommandInstaller); +# this default only covers a hand-driven run, and must stay the address archied +# serves /healthz on -- config.example.toml's [web] listen. +HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8484}" +HEALTH_URL="${HEALTH_URL%/}" REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" POLL_INTERVAL_SECONDS=3