fix(update): probe the dashboard's real address after a self-update

The watchdog polled http://127.0.0.1:8420/healthz. Nothing binds 8420: the
daemon serves /healthz on [web].listen (127.0.0.1:8484 by default), so every
dashboard- or Telegram-initiated update waited out the 60s health timeout and
rolled back a release that had come up fine.

The daemon is the only process that knows which address it binds, so it now
derives the probe URL from its own config and hands it to the installer;
CommandInstaller exports ARCHIE_HEALTH_URL and the install script forwards it
into the watchdog unit, whose remaining default matches the dashboard's.

The suite never dialled the URL: every watchdog test faked curl as an
immediate failure, so only the unhealthy branch existed. One test now runs
the real curl against a real listener, and another pins the script default.
This commit is contained in:
sam committed 2026-09-09 23:29:11 +10:00
1 parent 694aaeb7cd
commit 2fe708aa64
9 files changed
+212 -7

No files matched your search

+5 -1
View File
@@ -26,6 +26,7 @@ import (
"github.com/samcharles93/archie-core/internal/secret"
"github.com/samcharles93/archie-core/internal/store"
"github.com/samcharles93/archie-core/internal/tools"
"github.com/samcharles93/archie-core/internal/webui"
)
// telegramSetup contains the inputs needed to initialise the Telegram
@@ -235,7 +236,10 @@ func makeUpdateService(s telegramSetup) *releaseupdate.Service {
Enrich: componentInstallTypeEnricher(s.AgentStatus, cfg.NATS),
}
if len(cfg.Chat.Telegram.UpdateInstallCommand) != 0 {
updates.Installer = releaseupdate.CommandInstaller{Command: cfg.Chat.Telegram.UpdateInstallCommand}
updates.Installer = releaseupdate.CommandInstaller{
Command: cfg.Chat.Telegram.UpdateInstallCommand,
HealthURL: webui.HealthURL(cfg.Web.Listen),
}
}
return updates
}
@@ -200,3 +200,36 @@ func TestDaemonRunningVersionsIncludesObservedAgent(t *testing.T) {
t.Errorf("running[%q] = %q, want the observed version %q", releaseupdate.ComponentAgent, got, "1.9.11")
}
}
// TestMakeUpdateServiceDerivesHealthURLFromWebListen: the watchdog's health
// probe has to reach the dashboard listener this instance actually binds.
// Left to the script's own default it polls a port nothing serves, waits out
// the timeout and rolls back a release that came up fine (archie-core-1r4g).
func TestMakeUpdateServiceDerivesHealthURLFromWebListen(t *testing.T) {
for _, tc := range []struct{ name, listen, want string }{
{name: "loopback dashboard", listen: "127.0.0.1:9000", want: "http://127.0.0.1:9000"},
{name: "wildcard bind", listen: "0.0.0.0:8484", want: "http://localhost:8484"},
{name: "dashboard off", listen: "off", want: ""},
{name: "dashboard unset", listen: "", want: ""},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := config.Config{}
cfg.Web.Listen = tc.listen
cfg.Chat.Telegram.UpdateCheckCommand = []string{"archie-update-check"}
cfg.Chat.Telegram.UpdateInstallCommand = []string{"archie-update-install"}
service := makeUpdateService(telegramSetup{Cfg: config.NewHolder(cfg)})
if service == nil {
t.Fatal("makeUpdateService returned nil despite a configured check command")
}
installer, ok := service.Installer.(releaseupdate.CommandInstaller)
if !ok {
t.Fatalf("Service.Installer = %T, want releaseupdate.CommandInstaller", service.Installer)
}
if installer.HealthURL != tc.want {
t.Errorf("Installer.HealthURL = %q, want %q", installer.HealthURL, tc.want)
}
})
}
}
+12 -1
View File
@@ -36,7 +36,15 @@ func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) {
return snapshot, nil
}
type CommandInstaller struct{ Command []string }
// CommandInstaller runs the deployment's own install command. HealthURL is
// the base URL its restart tooling must poll for /healthz -- only this
// process knows which address it actually serves, so a script left to guess
// one rolls back releases that came up healthy (archie-core-1r4g). Empty
// leaves ARCHIE_HEALTH_URL as the environment already has it.
type CommandInstaller struct {
Command []string
HealthURL string
}
// Install runs the configured command and streams its stdout to progress
// line by line as the command emits it -- the earlier CombinedOutput-based
@@ -60,6 +68,9 @@ func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta I
"ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed,
"ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available,
)
if i.HealthURL != "" {
cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
+29
View File
@@ -151,3 +151,32 @@ func TestCommandInstallerEmptyCommandReturnsError(t *testing.T) {
t.Fatal("Install() error = nil, want empty-command error")
}
}
// TestCommandInstallerForwardsHealthURL: only the daemon knows which address
// it serves /healthz on, so it, not the install script, decides what the
// post-restart probe polls (archie-core-1r4g).
func TestCommandInstallerForwardsHealthURL(t *testing.T) {
script := writeFixtureScript(t, `
echo "health=${ARCHIE_HEALTH_URL:-unset}"
`)
t.Setenv("ARCHIE_HEALTH_URL", "http://operator.example:9999")
for _, tc := range []struct{ name, healthURL, want string }{
{name: "configured", healthURL: "http://127.0.0.1:9000", want: "health=http://127.0.0.1:9000"},
// An unset field must not clobber an operator's own export: a
// deployment with the dashboard off has no address to derive and
// the exported value is the only one there is.
{name: "unset leaves the environment alone", want: "health=http://operator.example:9999"},
} {
t.Run(tc.name, func(t *testing.T) {
installer := CommandInstaller{Command: []string{script}, HealthURL: tc.healthURL}
var progress []string
if _, err := installer.Install(context.Background(), Snapshot{}, InstallMeta{}, func(line string) { progress = append(progress, line) }); err != nil {
t.Fatalf("Install() error = %v", err)
}
if len(progress) != 1 || progress[0] != tc.want {
t.Fatalf("progress = %#v, want [%q]", progress, tc.want)
}
})
}
}
+88 -3
View File
@@ -4,6 +4,8 @@ import (
"bufio"
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
@@ -90,6 +92,21 @@ func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
}
}
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
// the two: systemd-run starts the watchdog with a clean environment, so a
// health address the daemon derived reaches the probe only if the install
// script hands it over explicitly (archie-core-1r4g).
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
_, calls := runUpdateInstallScript(t, map[string]string{
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
})
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
}
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
tests := []struct {
name string
@@ -125,6 +142,51 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent
}
}
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
// no other test reaches: a daemon that comes back up. Every other watchdog
// test fakes curl as an immediate failure, so before this one the probe URL
// the script actually builds was never dialled and its default could point
// anywhere (archie-core-1r4g).
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
daemon := httptest.NewServer(mux)
defer daemon.Close()
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
env: map[string]string{
"ARCHIE_HEALTH_URL": daemon.URL,
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
},
})
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
if installed != "new daemon" {
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
}
assertCallAbsent(t, calls, "docker image load")
}
// TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener pins the
// fallback the script uses when nothing exports one. It has to be the
// address archied serves /healthz on (config.example.toml's [web] listen);
// anything else times out and rolls back a healthy release.
func TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener(t *testing.T) {
report, _, calls := runUpdateWatchdog(t, watchdogRun{
components: "daemon",
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
})
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8484/healthz")
if report.HealthCheck != "passed" || report.RolledBack {
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
}
}
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
root, err := filepath.Abs(filepath.Join("..", ".."))
if err != nil {
@@ -343,7 +405,26 @@ func writeFakeCommand(t *testing.T, dir, name, body string) {
}
}
// watchdogRun configures one end-to-end run of the watchdog script. curl is
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
// which is what turns a health probe into a genuine test of the URL the
// watchdog builds rather than of a stub that ignores it.
type watchdogRun struct {
components string
curl string
env map[string]string
}
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
t.Helper()
return runUpdateWatchdog(t, watchdogRun{
components: components,
curl: `exit 1`,
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
})
}
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
t.Helper()
ctx := t.Context()
@@ -368,7 +449,9 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string,
}
}
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
if run.curl != "" {
writeFakeCommand(t, fakeDir, "curl", run.curl)
}
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
callsPath := filepath.Join(work, "calls")
@@ -380,13 +463,15 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string,
"ARCHIE_TEST_CALLS="+callsPath,
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
"ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
"ARCHIE_UPDATE_COMPONENTS="+components,
"ARCHIE_UPDATE_COMPONENTS="+run.components,
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
)
for key, value := range run.env {
cmd.Env = append(cmd.Env, key+"="+value)
}
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("watchdog failed: %v\n%s", err, output)
}
+18
View File
@@ -60,6 +60,24 @@ func DashboardURL(listen, token string) string {
return url
}
// HealthURL renders the base URL of this process's /healthz endpoint for
// out-of-process tooling: the update watchdog polls it after restarting
// archied to decide whether the release it just installed came up.
//
// It is empty when the dashboard is disabled, because /healthz is served by
// the dashboard listener and there is no other surface to probe.
//
// The address must remain the one archied itself binds. The watchdog is
// verifying archied's own restart, so probing an address served by a
// different process would report health for something it never restarted.
func HealthURL(listen string) string {
listen = strings.TrimSpace(listen)
if listen == "" || listen == "off" {
return ""
}
return strings.TrimSuffix(DashboardURL(listen, ""), "/")
}
// LoadOrCreateToken returns the dashboard token, generating and persisting one
// on first use. The token is created rather than configured so a non-loopback
// bind cannot be brought up unprotected by omission: there is no setup step to
+21
View File
@@ -15,3 +15,24 @@ func TestDashboardURL(t *testing.T) {
}
}
}
// TestHealthURL pins what out-of-process tooling probes after restarting
// archied. A wrong address here reads as an unhealthy release and triggers a
// rollback of one that came up fine (archie-core-1r4g).
func TestHealthURL(t *testing.T) {
for _, tc := range []struct{ listen, want string }{
{"127.0.0.1:8484", "http://127.0.0.1:8484"},
{"localhost:8484", "http://localhost:8484"},
{"0.0.0.0:8484", "http://localhost:8484"},
{":8484", "http://localhost:8484"},
{"[::]:8484", "http://localhost:8484"},
{"100.64.1.2:9000", "http://100.64.1.2:9000"},
{" 127.0.0.1:8484 ", "http://127.0.0.1:8484"},
{"off", ""},
{"", ""},
} {
if got := HealthURL(tc.listen); got != tc.want {
t.Errorf("HealthURL(%q) = %q, want %q", tc.listen, got, tc.want)
}
}
}
+1 -1
View File
@@ -166,7 +166,7 @@ fi
echo "==> handing off to the update watchdog for restart and health verification"
systemd-run --user --unit=archied-update-restart --collect --quiet \
--setenv=ARCHIE_BIN_DIR="$BIN_DIR" \
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" \
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \
--setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \
--setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
+5 -1
View File
@@ -12,7 +12,11 @@ AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}"
# The daemon exports its own dashboard address (releaseupdate.CommandInstaller);
# this default only covers a hand-driven run, and must stay the address archied
# serves /healthz on -- config.example.toml's [web] listen.
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8484}"
HEALTH_URL="${HEALTH_URL%/}"
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
POLL_INTERVAL_SECONDS=3