fix(update): probe the dashboard's real address after a self-update
The watchdog polled http://127.0.0.1:8420/healthz. Nothing binds 8420: the daemon serves /healthz on [web].listen (127.0.0.1:8484 by default), so every dashboard- or Telegram-initiated update waited out the 60s health timeout and rolled back a release that had come up fine. The daemon is the only process that knows which address it binds, so it now derives the probe URL from its own config and hands it to the installer; CommandInstaller exports ARCHIE_HEALTH_URL and the install script forwards it into the watchdog unit, whose remaining default matches the dashboard's. The suite never dialled the URL: every watchdog test faked curl as an immediate failure, so only the unhealthy branch existed. One test now runs the real curl against a real listener, and another pins the script default.
This commit is contained in:
1 parent
694aaeb7cd
commit
2fe708aa64
9 files changed
+212
-7
No files matched your search
@@ -26,6 +26,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
"github.com/samcharles93/archie-core/internal/tools"
|
||||
"github.com/samcharles93/archie-core/internal/webui"
|
||||
)
|
||||
|
||||
// telegramSetup contains the inputs needed to initialise the Telegram
|
||||
@@ -235,7 +236,10 @@ func makeUpdateService(s telegramSetup) *releaseupdate.Service {
|
||||
Enrich: componentInstallTypeEnricher(s.AgentStatus, cfg.NATS),
|
||||
}
|
||||
if len(cfg.Chat.Telegram.UpdateInstallCommand) != 0 {
|
||||
updates.Installer = releaseupdate.CommandInstaller{Command: cfg.Chat.Telegram.UpdateInstallCommand}
|
||||
updates.Installer = releaseupdate.CommandInstaller{
|
||||
Command: cfg.Chat.Telegram.UpdateInstallCommand,
|
||||
HealthURL: webui.HealthURL(cfg.Web.Listen),
|
||||
}
|
||||
}
|
||||
return updates
|
||||
}
|
||||
|
||||
@@ -200,3 +200,36 @@ func TestDaemonRunningVersionsIncludesObservedAgent(t *testing.T) {
|
||||
t.Errorf("running[%q] = %q, want the observed version %q", releaseupdate.ComponentAgent, got, "1.9.11")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMakeUpdateServiceDerivesHealthURLFromWebListen: the watchdog's health
|
||||
// probe has to reach the dashboard listener this instance actually binds.
|
||||
// Left to the script's own default it polls a port nothing serves, waits out
|
||||
// the timeout and rolls back a release that came up fine (archie-core-1r4g).
|
||||
func TestMakeUpdateServiceDerivesHealthURLFromWebListen(t *testing.T) {
|
||||
for _, tc := range []struct{ name, listen, want string }{
|
||||
{name: "loopback dashboard", listen: "127.0.0.1:9000", want: "http://127.0.0.1:9000"},
|
||||
{name: "wildcard bind", listen: "0.0.0.0:8484", want: "http://localhost:8484"},
|
||||
{name: "dashboard off", listen: "off", want: ""},
|
||||
{name: "dashboard unset", listen: "", want: ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := config.Config{}
|
||||
cfg.Web.Listen = tc.listen
|
||||
cfg.Chat.Telegram.UpdateCheckCommand = []string{"archie-update-check"}
|
||||
cfg.Chat.Telegram.UpdateInstallCommand = []string{"archie-update-install"}
|
||||
|
||||
service := makeUpdateService(telegramSetup{Cfg: config.NewHolder(cfg)})
|
||||
|
||||
if service == nil {
|
||||
t.Fatal("makeUpdateService returned nil despite a configured check command")
|
||||
}
|
||||
installer, ok := service.Installer.(releaseupdate.CommandInstaller)
|
||||
if !ok {
|
||||
t.Fatalf("Service.Installer = %T, want releaseupdate.CommandInstaller", service.Installer)
|
||||
}
|
||||
if installer.HealthURL != tc.want {
|
||||
t.Errorf("Installer.HealthURL = %q, want %q", installer.HealthURL, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -36,7 +36,15 @@ func (c CommandCatalog) Check(ctx context.Context) (Snapshot, error) {
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
type CommandInstaller struct{ Command []string }
|
||||
// CommandInstaller runs the deployment's own install command. HealthURL is
|
||||
// the base URL its restart tooling must poll for /healthz -- only this
|
||||
// process knows which address it actually serves, so a script left to guess
|
||||
// one rolls back releases that came up healthy (archie-core-1r4g). Empty
|
||||
// leaves ARCHIE_HEALTH_URL as the environment already has it.
|
||||
type CommandInstaller struct {
|
||||
Command []string
|
||||
HealthURL string
|
||||
}
|
||||
|
||||
// Install runs the configured command and streams its stdout to progress
|
||||
// line by line as the command emits it -- the earlier CombinedOutput-based
|
||||
@@ -60,6 +68,9 @@ func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta I
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed,
|
||||
"ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available,
|
||||
)
|
||||
if i.HealthURL != "" {
|
||||
cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL)
|
||||
}
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
|
||||
@@ -151,3 +151,32 @@ func TestCommandInstallerEmptyCommandReturnsError(t *testing.T) {
|
||||
t.Fatal("Install() error = nil, want empty-command error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCommandInstallerForwardsHealthURL: only the daemon knows which address
|
||||
// it serves /healthz on, so it, not the install script, decides what the
|
||||
// post-restart probe polls (archie-core-1r4g).
|
||||
func TestCommandInstallerForwardsHealthURL(t *testing.T) {
|
||||
script := writeFixtureScript(t, `
|
||||
echo "health=${ARCHIE_HEALTH_URL:-unset}"
|
||||
`)
|
||||
t.Setenv("ARCHIE_HEALTH_URL", "http://operator.example:9999")
|
||||
|
||||
for _, tc := range []struct{ name, healthURL, want string }{
|
||||
{name: "configured", healthURL: "http://127.0.0.1:9000", want: "health=http://127.0.0.1:9000"},
|
||||
// An unset field must not clobber an operator's own export: a
|
||||
// deployment with the dashboard off has no address to derive and
|
||||
// the exported value is the only one there is.
|
||||
{name: "unset leaves the environment alone", want: "health=http://operator.example:9999"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
installer := CommandInstaller{Command: []string{script}, HealthURL: tc.healthURL}
|
||||
var progress []string
|
||||
if _, err := installer.Install(context.Background(), Snapshot{}, InstallMeta{}, func(line string) { progress = append(progress, line) }); err != nil {
|
||||
t.Fatalf("Install() error = %v", err)
|
||||
}
|
||||
if len(progress) != 1 || progress[0] != tc.want {
|
||||
t.Fatalf("progress = %#v, want [%q]", progress, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -90,6 +92,21 @@ func TestUpdateInstallNoopDoesNotFetchBuildInstallOrRestart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateInstallForwardsHealthURLToTheWatchdogUnit closes the link between
|
||||
// the two: systemd-run starts the watchdog with a clean environment, so a
|
||||
// health address the daemon derived reaches the probe only if the install
|
||||
// script hands it over explicitly (archie-core-1r4g).
|
||||
func TestUpdateInstallForwardsHealthURLToTheWatchdogUnit(t *testing.T) {
|
||||
_, calls := runUpdateInstallScript(t, map[string]string{
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS": "1.12.0",
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION": "1.13.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS": "1.9.9",
|
||||
"ARCHIE_HEALTH_URL": "http://127.0.0.1:9000",
|
||||
})
|
||||
|
||||
assertCallContains(t, calls, "systemd-run ", "--setenv=ARCHIE_HEALTH_URL=http://127.0.0.1:9000")
|
||||
}
|
||||
|
||||
func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponents(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -125,6 +142,51 @@ func TestUpdateWatchdogRollsBackManagedWorkerImageAndReportsOnlyChangedComponent
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateWatchdogPassesHealthCheckAgainstRealListener exercises the branch
|
||||
// no other test reaches: a daemon that comes back up. Every other watchdog
|
||||
// test fakes curl as an immediate failure, so before this one the probe URL
|
||||
// the script actually builds was never dialled and its default could point
|
||||
// anywhere (archie-core-1r4g).
|
||||
func TestUpdateWatchdogPassesHealthCheckAgainstRealListener(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
|
||||
daemon := httptest.NewServer(mux)
|
||||
defer daemon.Close()
|
||||
|
||||
report, installed, calls := runUpdateWatchdog(t, watchdogRun{
|
||||
components: "daemon",
|
||||
env: map[string]string{
|
||||
"ARCHIE_HEALTH_URL": daemon.URL,
|
||||
"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10",
|
||||
},
|
||||
})
|
||||
|
||||
if report.HealthCheck != "passed" || report.RolledBack {
|
||||
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
||||
}
|
||||
if installed != "new daemon" {
|
||||
t.Fatalf("installed daemon = %q, want the new binary left in place", installed)
|
||||
}
|
||||
assertCallAbsent(t, calls, "docker image load")
|
||||
}
|
||||
|
||||
// TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener pins the
|
||||
// fallback the script uses when nothing exports one. It has to be the
|
||||
// address archied serves /healthz on (config.example.toml's [web] listen);
|
||||
// anything else times out and rolls back a healthy release.
|
||||
func TestUpdateWatchdogDefaultHealthURLTargetsTheDashboardListener(t *testing.T) {
|
||||
report, _, calls := runUpdateWatchdog(t, watchdogRun{
|
||||
components: "daemon",
|
||||
curl: `printf '%s\n' "curl $*" >> "$ARCHIE_TEST_CALLS"; exit 0`,
|
||||
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "10"},
|
||||
})
|
||||
|
||||
assertCallContains(t, calls, "curl ", "http://127.0.0.1:8484/healthz")
|
||||
if report.HealthCheck != "passed" || report.RolledBack {
|
||||
t.Fatalf("report = %#v, want a passed health check with no rollback", report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateWatchdogRestoresTaskDatabaseOnDaemonRollback(t *testing.T) {
|
||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
||||
if err != nil {
|
||||
@@ -343,7 +405,26 @@ func writeFakeCommand(t *testing.T, dir, name, body string) {
|
||||
}
|
||||
}
|
||||
|
||||
// watchdogRun configures one end-to-end run of the watchdog script. curl is
|
||||
// the fake curl placed on PATH; leaving it empty keeps the real curl there,
|
||||
// which is what turns a health probe into a genuine test of the URL the
|
||||
// watchdog builds rather than of a stub that ignores it.
|
||||
type watchdogRun struct {
|
||||
components string
|
||||
curl string
|
||||
env map[string]string
|
||||
}
|
||||
|
||||
func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string, []string) {
|
||||
t.Helper()
|
||||
return runUpdateWatchdog(t, watchdogRun{
|
||||
components: components,
|
||||
curl: `exit 1`,
|
||||
env: map[string]string{"ARCHIE_UPDATE_HEALTH_TIMEOUT": "0"},
|
||||
})
|
||||
}
|
||||
|
||||
func runUpdateWatchdog(t *testing.T, run watchdogRun) (Report, string, []string) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -368,7 +449,9 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string,
|
||||
}
|
||||
}
|
||||
writeFakeCommand(t, fakeDir, "systemctl", `exit 0`)
|
||||
writeFakeCommand(t, fakeDir, "curl", `exit 1`)
|
||||
if run.curl != "" {
|
||||
writeFakeCommand(t, fakeDir, "curl", run.curl)
|
||||
}
|
||||
writeFakeCommand(t, fakeDir, "docker", `printf '%s\n' "docker $*" >> "$ARCHIE_TEST_CALLS"`)
|
||||
callsPath := filepath.Join(work, "calls")
|
||||
|
||||
@@ -380,13 +463,15 @@ func runUpdateWatchdogFailure(t *testing.T, components string) (Report, string,
|
||||
"ARCHIE_TEST_CALLS="+callsPath,
|
||||
"ARCHIE_AGENT_IMAGE=registry.example/archie-agent:stable",
|
||||
"ARCHIE_UPDATE_REPORT_PATH="+reportPath,
|
||||
"ARCHIE_UPDATE_HEALTH_TIMEOUT=0",
|
||||
"ARCHIE_UPDATE_COMPONENTS="+components,
|
||||
"ARCHIE_UPDATE_COMPONENTS="+run.components,
|
||||
"ARCHIE_UPDATE_PREVIOUS_GATEWAY=1.12.0",
|
||||
"ARCHIE_UPDATE_INSTALLED_GATEWAY=1.13.0",
|
||||
"ARCHIE_UPDATE_PREVIOUS_RUNTIME=1.9.9",
|
||||
"ARCHIE_UPDATE_INSTALLED_RUNTIME=1.10.0",
|
||||
)
|
||||
for key, value := range run.env {
|
||||
cmd.Env = append(cmd.Env, key+"="+value)
|
||||
}
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("watchdog failed: %v\n%s", err, output)
|
||||
}
|
||||
|
||||
@@ -60,6 +60,24 @@ func DashboardURL(listen, token string) string {
|
||||
return url
|
||||
}
|
||||
|
||||
// HealthURL renders the base URL of this process's /healthz endpoint for
|
||||
// out-of-process tooling: the update watchdog polls it after restarting
|
||||
// archied to decide whether the release it just installed came up.
|
||||
//
|
||||
// It is empty when the dashboard is disabled, because /healthz is served by
|
||||
// the dashboard listener and there is no other surface to probe.
|
||||
//
|
||||
// The address must remain the one archied itself binds. The watchdog is
|
||||
// verifying archied's own restart, so probing an address served by a
|
||||
// different process would report health for something it never restarted.
|
||||
func HealthURL(listen string) string {
|
||||
listen = strings.TrimSpace(listen)
|
||||
if listen == "" || listen == "off" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSuffix(DashboardURL(listen, ""), "/")
|
||||
}
|
||||
|
||||
// LoadOrCreateToken returns the dashboard token, generating and persisting one
|
||||
// on first use. The token is created rather than configured so a non-loopback
|
||||
// bind cannot be brought up unprotected by omission: there is no setup step to
|
||||
|
||||
@@ -15,3 +15,24 @@ func TestDashboardURL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealthURL pins what out-of-process tooling probes after restarting
|
||||
// archied. A wrong address here reads as an unhealthy release and triggers a
|
||||
// rollback of one that came up fine (archie-core-1r4g).
|
||||
func TestHealthURL(t *testing.T) {
|
||||
for _, tc := range []struct{ listen, want string }{
|
||||
{"127.0.0.1:8484", "http://127.0.0.1:8484"},
|
||||
{"localhost:8484", "http://localhost:8484"},
|
||||
{"0.0.0.0:8484", "http://localhost:8484"},
|
||||
{":8484", "http://localhost:8484"},
|
||||
{"[::]:8484", "http://localhost:8484"},
|
||||
{"100.64.1.2:9000", "http://100.64.1.2:9000"},
|
||||
{" 127.0.0.1:8484 ", "http://127.0.0.1:8484"},
|
||||
{"off", ""},
|
||||
{"", ""},
|
||||
} {
|
||||
if got := HealthURL(tc.listen); got != tc.want {
|
||||
t.Errorf("HealthURL(%q) = %q, want %q", tc.listen, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -166,7 +166,7 @@ fi
|
||||
echo "==> handing off to the update watchdog for restart and health verification"
|
||||
systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_BIN_DIR="$BIN_DIR" \
|
||||
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}" \
|
||||
--setenv=ARCHIE_HEALTH_URL="${ARCHIE_HEALTH_URL:-}" \
|
||||
--setenv=ARCHIE_UPDATE_REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" \
|
||||
--setenv=ARCHIE_UPDATE_CHANNEL="${ARCHIE_UPDATE_CHANNEL:-}" \
|
||||
--setenv=ARCHIE_UPDATE_CHAT_ID="${ARCHIE_UPDATE_CHAT_ID:-0}" \
|
||||
|
||||
@@ -12,7 +12,11 @@ AGENT_IMAGE="${ARCHIE_AGENT_IMAGE:-ghcr.io/samcharles93/archie-agent:latest}"
|
||||
AGENT_IMAGE_BACKUP="${ARCHIE_AGENT_IMAGE_BACKUP:-$BIN_DIR/archie-agent-image.prev.tar}"
|
||||
TASK_DB_PATH="${ARCHIE_TASK_DB_PATH:-}"
|
||||
TASK_DB_BACKUP="${ARCHIE_TASK_DB_BACKUP:-$BIN_DIR/archie-task-db.prev.sqlite}"
|
||||
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8420}"
|
||||
# The daemon exports its own dashboard address (releaseupdate.CommandInstaller);
|
||||
# this default only covers a hand-driven run, and must stay the address archied
|
||||
# serves /healthz on -- config.example.toml's [web] listen.
|
||||
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8484}"
|
||||
HEALTH_URL="${HEALTH_URL%/}"
|
||||
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
||||
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
||||
POLL_INTERVAL_SECONDS=3
|
||||
|
||||
Reference in new issue
Block a user