fix(releaseupdate): run the release's watchdog and bound control-plane calls

This commit is contained in:
sam committed 2026-10-17 17:00:04 +11:00
1 parent 5c8e8b0fcb
commit a973bd6567
4 files changed
+43 -15

No files matched your search

+1 -1
View File
@@ -83,7 +83,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
return 1
}
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta,
Version: *version, ReleaseURL: *releaseURL, Meta: meta,
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
return checkUpdateTopology(ctx, *configPath, required, managed, out)
},
+4 -4
View File
@@ -38,9 +38,7 @@ type PrepareRequest struct {
Version string
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
ReleaseURL string
// Self is the archied executable that will run the watchdog.
Self string
Meta InstallMeta
Meta InstallMeta
// CheckTopology refuses a host missing a unit or config section the
// release requires. managed reports whether units are observable.
CheckTopology func(ctx context.Context, required Topology, managed bool) error
@@ -186,7 +184,9 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error
if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil {
return err
}
if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
// The release's own archied runs the watchdog, so it speaks the protocol
// of the services it switches to.
if err := installFile(filepath.Join(release, WatchdogBinary), filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
return fmt.Errorf("freeze the watchdog: %w", err)
}
for _, unit := range j.Units {
+14 -5
View File
@@ -306,7 +306,9 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
if j.RestoredDatabase {
return true, nil
}
schema, err := t.Database.SchemaVersion(ctx)
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
schema, err := t.Database.SchemaVersion(callCtx)
cancel()
if err != nil {
return false, fmt.Errorf("read schema version: %w", err)
}
@@ -319,7 +321,8 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
return true, nil
}
// setImage retries while the State Store comes up.
// setImage retries while the State Store comes up. No attempt outlives the
// deadline, so a peer that never answers still ends in rollback.
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
if j.Managed {
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
@@ -328,7 +331,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
}
deadline := time.Now().Add(t.HealthTimeout)
for {
err := t.Policy.SetImage(ctx, expected, desired)
callCtx, cancel := context.WithDeadline(ctx, deadline)
err := t.Policy.SetImage(callCtx, expected, desired)
cancel()
if err == nil || time.Now().After(deadline) {
return err
}
@@ -341,7 +346,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
func (t *Transaction) observe(ctx context.Context, version string) error {
deadline := time.Now().Add(t.HealthTimeout)
for {
reported, err := t.Health.Healthy(ctx)
callCtx, cancel := context.WithDeadline(ctx, deadline)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err == nil && sameVersion(reported, version) {
break
}
@@ -358,7 +365,9 @@ func (t *Transaction) observe(ctx context.Context, version string) error {
if err := sleep(ctx, t.Poll); err != nil {
return err
}
reported, err := t.Health.Healthy(ctx)
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err != nil || !sameVersion(reported, version) {
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
}
+24 -5
View File
@@ -32,6 +32,7 @@ type fakeHost struct {
managed bool
migrates bool
unhealthy bool
hangs bool // the State Store never answers the switch to the candidate
image string
backedUp string
schema int64
@@ -59,7 +60,11 @@ func (h *fakeHost) Pull(_ context.Context, ref string) error {
func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil }
func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error {
func (h *fakeHost) SetImage(ctx context.Context, expected, desired string) error {
if h.hangs && desired == newImage {
<-ctx.Done()
return ctx.Err()
}
if h.image != desired && h.image != expected {
return errors.New("image policy changed")
}
@@ -147,6 +152,7 @@ func TestTransaction(t *testing.T) {
{name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}},
{name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old},
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
{name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old},
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
@@ -171,8 +177,6 @@ func TestTransaction(t *testing.T) {
for _, unit := range testUnits {
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
}
self := filepath.Join(root, "self")
writeTestFile(t, self, "frozen watchdog")
report := filepath.Join(root, "report.json")
tx := &Transaction{
StateDir: filepath.Join(root, "state"), BinDir: host.bin,
@@ -180,7 +184,7 @@ func TestTransaction(t *testing.T) {
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
}
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
switch {
case tt.prepare != "":
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
@@ -188,11 +192,17 @@ func TestTransaction(t *testing.T) {
}
case err != nil:
t.Fatalf("Prepare: %v", err)
case !strings.Contains(readTestFile(t, filepath.Join(JournalPath(tx.StateDir), WatchdogBinary)), " 2.0.0"):
t.Fatal("the frozen watchdog is not the release's archied")
case tt.crashAt != "":
crash(t, tx, &host, tt.crashAt)
err = tx.Run(t.Context(), true)
default:
err = tx.Run(t.Context(), false)
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
if err = tx.Run(ctx, false); ctx.Err() != nil {
t.Fatal("Run outlived HealthTimeout")
}
}
if tt.prepare == "" && err != nil {
t.Fatalf("Run: %v", err)
@@ -286,3 +296,12 @@ func writeTestFile(t *testing.T, path, data string) {
t.Fatal(err)
}
}
func readTestFile(t *testing.T, path string) string {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
return string(data)
}