From a973bd65674c23f116a8fb421850550c5b0d3fae Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 17 Oct 2026 16:59:44 +1100 Subject: [PATCH] fix(releaseupdate): run the release's watchdog and bound control-plane calls --- internal/app/archied/update.go | 2 +- internal/releaseupdate/prepare.go | 8 +++--- internal/releaseupdate/transaction.go | 19 ++++++++++---- internal/releaseupdate/transaction_test.go | 29 ++++++++++++++++++---- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/internal/app/archied/update.go b/internal/app/archied/update.go index 7fd530b85..1a66120b8 100644 --- a/internal/app/archied/update.go +++ b/internal/app/archied/update.go @@ -83,7 +83,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int { return 1 } result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{ - Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta, + Version: *version, ReleaseURL: *releaseURL, Meta: meta, CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error { return checkUpdateTopology(ctx, *configPath, required, managed, out) }, diff --git a/internal/releaseupdate/prepare.go b/internal/releaseupdate/prepare.go index 65e93e920..2eff63659 100644 --- a/internal/releaseupdate/prepare.go +++ b/internal/releaseupdate/prepare.go @@ -38,9 +38,7 @@ type PrepareRequest struct { Version string // ReleaseURL is the base the release's zip and SHA256SUMS are under. ReleaseURL string - // Self is the archied executable that will run the watchdog. - Self string - Meta InstallMeta + Meta InstallMeta // CheckTopology refuses a host missing a unit or config section the // release requires. managed reports whether units are observable. CheckTopology func(ctx context.Context, required Topology, managed bool) error @@ -186,7 +184,9 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil { return err } - if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil { + // The release's own archied runs the watchdog, so it speaks the protocol + // of the services it switches to. + if err := installFile(filepath.Join(release, WatchdogBinary), filepath.Join(preparing, WatchdogBinary), 0o700); err != nil { return fmt.Errorf("freeze the watchdog: %w", err) } for _, unit := range j.Units { diff --git a/internal/releaseupdate/transaction.go b/internal/releaseupdate/transaction.go index 3a07cfbfc..c92375737 100644 --- a/internal/releaseupdate/transaction.go +++ b/internal/releaseupdate/transaction.go @@ -306,7 +306,9 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err if j.RestoredDatabase { return true, nil } - schema, err := t.Database.SchemaVersion(ctx) + callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout) + schema, err := t.Database.SchemaVersion(callCtx) + cancel() if err != nil { return false, fmt.Errorf("read schema version: %w", err) } @@ -319,7 +321,8 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err return true, nil } -// setImage retries while the State Store comes up. +// setImage retries while the State Store comes up. No attempt outlives the +// deadline, so a peer that never answers still ends in rollback. func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error { if j.Managed { if err := t.Units.Start(ctx, stateStoreUnit); err != nil { @@ -328,7 +331,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired } deadline := time.Now().Add(t.HealthTimeout) for { - err := t.Policy.SetImage(ctx, expected, desired) + callCtx, cancel := context.WithDeadline(ctx, deadline) + err := t.Policy.SetImage(callCtx, expected, desired) + cancel() if err == nil || time.Now().After(deadline) { return err } @@ -341,7 +346,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired func (t *Transaction) observe(ctx context.Context, version string) error { deadline := time.Now().Add(t.HealthTimeout) for { - reported, err := t.Health.Healthy(ctx) + callCtx, cancel := context.WithDeadline(ctx, deadline) + reported, err := t.Health.Healthy(callCtx) + cancel() if err == nil && sameVersion(reported, version) { break } @@ -358,7 +365,9 @@ func (t *Transaction) observe(ctx context.Context, version string) error { if err := sleep(ctx, t.Poll); err != nil { return err } - reported, err := t.Health.Healthy(ctx) + callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout) + reported, err := t.Health.Healthy(callCtx) + cancel() if err != nil || !sameVersion(reported, version) { return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err)) } diff --git a/internal/releaseupdate/transaction_test.go b/internal/releaseupdate/transaction_test.go index 7429e2920..d5b9c6ce4 100644 --- a/internal/releaseupdate/transaction_test.go +++ b/internal/releaseupdate/transaction_test.go @@ -32,6 +32,7 @@ type fakeHost struct { managed bool migrates bool unhealthy bool + hangs bool // the State Store never answers the switch to the candidate image string backedUp string schema int64 @@ -59,7 +60,11 @@ func (h *fakeHost) Pull(_ context.Context, ref string) error { func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil } -func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error { +func (h *fakeHost) SetImage(ctx context.Context, expected, desired string) error { + if h.hangs && desired == newImage { + <-ctx.Done() + return ctx.Err() + } if h.image != desired && h.image != expected { return errors.New("image policy changed") } @@ -147,6 +152,7 @@ func TestTransaction(t *testing.T) { {name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}}, {name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old}, {name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}}, + {name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old}, {name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}}, {name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}}, {name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}}, @@ -171,8 +177,6 @@ func TestTransaction(t *testing.T) { for _, unit := range testUnits { writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0")) } - self := filepath.Join(root, "self") - writeTestFile(t, self, "frozen watchdog") report := filepath.Join(root, "report.json") tx := &Transaction{ StateDir: filepath.Join(root, "state"), BinDir: host.bin, @@ -180,7 +184,7 @@ func TestTransaction(t *testing.T) { HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond, } - _, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}}) + _, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}}) switch { case tt.prepare != "": if err == nil || !strings.Contains(err.Error(), tt.prepare) { @@ -188,11 +192,17 @@ func TestTransaction(t *testing.T) { } case err != nil: t.Fatalf("Prepare: %v", err) + case !strings.Contains(readTestFile(t, filepath.Join(JournalPath(tx.StateDir), WatchdogBinary)), " 2.0.0"): + t.Fatal("the frozen watchdog is not the release's archied") case tt.crashAt != "": crash(t, tx, &host, tt.crashAt) err = tx.Run(t.Context(), true) default: - err = tx.Run(t.Context(), false) + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err = tx.Run(ctx, false); ctx.Err() != nil { + t.Fatal("Run outlived HealthTimeout") + } } if tt.prepare == "" && err != nil { t.Fatalf("Run: %v", err) @@ -286,3 +296,12 @@ func writeTestFile(t *testing.T, path, data string) { t.Fatal(err) } } + +func readTestFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +}