fix(releaseupdate): run the release's watchdog and bound control-plane calls
This commit is contained in:
4 files changed
+43
-15
No files matched your search
@@ -83,7 +83,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
|
||||
return 1
|
||||
}
|
||||
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
|
||||
Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta,
|
||||
Version: *version, ReleaseURL: *releaseURL, Meta: meta,
|
||||
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
|
||||
return checkUpdateTopology(ctx, *configPath, required, managed, out)
|
||||
},
|
||||
|
||||
@@ -38,9 +38,7 @@ type PrepareRequest struct {
|
||||
Version string
|
||||
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
|
||||
ReleaseURL string
|
||||
// Self is the archied executable that will run the watchdog.
|
||||
Self string
|
||||
Meta InstallMeta
|
||||
Meta InstallMeta
|
||||
// CheckTopology refuses a host missing a unit or config section the
|
||||
// release requires. managed reports whether units are observable.
|
||||
CheckTopology func(ctx context.Context, required Topology, managed bool) error
|
||||
@@ -186,7 +184,9 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error
|
||||
if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
|
||||
// The release's own archied runs the watchdog, so it speaks the protocol
|
||||
// of the services it switches to.
|
||||
if err := installFile(filepath.Join(release, WatchdogBinary), filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
|
||||
return fmt.Errorf("freeze the watchdog: %w", err)
|
||||
}
|
||||
for _, unit := range j.Units {
|
||||
|
||||
@@ -306,7 +306,9 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
|
||||
if j.RestoredDatabase {
|
||||
return true, nil
|
||||
}
|
||||
schema, err := t.Database.SchemaVersion(ctx)
|
||||
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
||||
schema, err := t.Database.SchemaVersion(callCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
@@ -319,7 +321,8 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// setImage retries while the State Store comes up.
|
||||
// setImage retries while the State Store comes up. No attempt outlives the
|
||||
// deadline, so a peer that never answers still ends in rollback.
|
||||
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
|
||||
if j.Managed {
|
||||
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
|
||||
@@ -328,7 +331,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
|
||||
}
|
||||
deadline := time.Now().Add(t.HealthTimeout)
|
||||
for {
|
||||
err := t.Policy.SetImage(ctx, expected, desired)
|
||||
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
err := t.Policy.SetImage(callCtx, expected, desired)
|
||||
cancel()
|
||||
if err == nil || time.Now().After(deadline) {
|
||||
return err
|
||||
}
|
||||
@@ -341,7 +346,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
|
||||
func (t *Transaction) observe(ctx context.Context, version string) error {
|
||||
deadline := time.Now().Add(t.HealthTimeout)
|
||||
for {
|
||||
reported, err := t.Health.Healthy(ctx)
|
||||
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||
reported, err := t.Health.Healthy(callCtx)
|
||||
cancel()
|
||||
if err == nil && sameVersion(reported, version) {
|
||||
break
|
||||
}
|
||||
@@ -358,7 +365,9 @@ func (t *Transaction) observe(ctx context.Context, version string) error {
|
||||
if err := sleep(ctx, t.Poll); err != nil {
|
||||
return err
|
||||
}
|
||||
reported, err := t.Health.Healthy(ctx)
|
||||
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
||||
reported, err := t.Health.Healthy(callCtx)
|
||||
cancel()
|
||||
if err != nil || !sameVersion(reported, version) {
|
||||
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ type fakeHost struct {
|
||||
managed bool
|
||||
migrates bool
|
||||
unhealthy bool
|
||||
hangs bool // the State Store never answers the switch to the candidate
|
||||
image string
|
||||
backedUp string
|
||||
schema int64
|
||||
@@ -59,7 +60,11 @@ func (h *fakeHost) Pull(_ context.Context, ref string) error {
|
||||
|
||||
func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil }
|
||||
|
||||
func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error {
|
||||
func (h *fakeHost) SetImage(ctx context.Context, expected, desired string) error {
|
||||
if h.hangs && desired == newImage {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if h.image != desired && h.image != expected {
|
||||
return errors.New("image policy changed")
|
||||
}
|
||||
@@ -147,6 +152,7 @@ func TestTransaction(t *testing.T) {
|
||||
{name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}},
|
||||
{name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old},
|
||||
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
|
||||
{name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old},
|
||||
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
|
||||
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
|
||||
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
|
||||
@@ -171,8 +177,6 @@ func TestTransaction(t *testing.T) {
|
||||
for _, unit := range testUnits {
|
||||
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
|
||||
}
|
||||
self := filepath.Join(root, "self")
|
||||
writeTestFile(t, self, "frozen watchdog")
|
||||
report := filepath.Join(root, "report.json")
|
||||
tx := &Transaction{
|
||||
StateDir: filepath.Join(root, "state"), BinDir: host.bin,
|
||||
@@ -180,7 +184,7 @@ func TestTransaction(t *testing.T) {
|
||||
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
|
||||
}
|
||||
|
||||
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
|
||||
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
|
||||
switch {
|
||||
case tt.prepare != "":
|
||||
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
|
||||
@@ -188,11 +192,17 @@ func TestTransaction(t *testing.T) {
|
||||
}
|
||||
case err != nil:
|
||||
t.Fatalf("Prepare: %v", err)
|
||||
case !strings.Contains(readTestFile(t, filepath.Join(JournalPath(tx.StateDir), WatchdogBinary)), " 2.0.0"):
|
||||
t.Fatal("the frozen watchdog is not the release's archied")
|
||||
case tt.crashAt != "":
|
||||
crash(t, tx, &host, tt.crashAt)
|
||||
err = tx.Run(t.Context(), true)
|
||||
default:
|
||||
err = tx.Run(t.Context(), false)
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
if err = tx.Run(ctx, false); ctx.Err() != nil {
|
||||
t.Fatal("Run outlived HealthTimeout")
|
||||
}
|
||||
}
|
||||
if tt.prepare == "" && err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
@@ -286,3 +296,12 @@ func writeTestFile(t *testing.T, path, data string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func readTestFile(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
Reference in new issue
Block a user