fix(releaseupdate): run the release's watchdog and bound control-plane calls
This commit is contained in:
4 files changed
+43
-15
No files matched your search
@@ -83,7 +83,7 @@ func runUpdateInstall(args []string, out, stderr io.Writer) int {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
|
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
|
||||||
Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta,
|
Version: *version, ReleaseURL: *releaseURL, Meta: meta,
|
||||||
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
|
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
|
||||||
return checkUpdateTopology(ctx, *configPath, required, managed, out)
|
return checkUpdateTopology(ctx, *configPath, required, managed, out)
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -38,9 +38,7 @@ type PrepareRequest struct {
|
|||||||
Version string
|
Version string
|
||||||
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
|
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
|
||||||
ReleaseURL string
|
ReleaseURL string
|
||||||
// Self is the archied executable that will run the watchdog.
|
Meta InstallMeta
|
||||||
Self string
|
|
||||||
Meta InstallMeta
|
|
||||||
// CheckTopology refuses a host missing a unit or config section the
|
// CheckTopology refuses a host missing a unit or config section the
|
||||||
// release requires. managed reports whether units are observable.
|
// release requires. managed reports whether units are observable.
|
||||||
CheckTopology func(ctx context.Context, required Topology, managed bool) error
|
CheckTopology func(ctx context.Context, required Topology, managed bool) error
|
||||||
@@ -186,7 +184,9 @@ func (t *Transaction) stage(req PrepareRequest, release string, j journal) error
|
|||||||
if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil {
|
if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
|
// The release's own archied runs the watchdog, so it speaks the protocol
|
||||||
|
// of the services it switches to.
|
||||||
|
if err := installFile(filepath.Join(release, WatchdogBinary), filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
|
||||||
return fmt.Errorf("freeze the watchdog: %w", err)
|
return fmt.Errorf("freeze the watchdog: %w", err)
|
||||||
}
|
}
|
||||||
for _, unit := range j.Units {
|
for _, unit := range j.Units {
|
||||||
|
|||||||
@@ -306,7 +306,9 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
|
|||||||
if j.RestoredDatabase {
|
if j.RestoredDatabase {
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
schema, err := t.Database.SchemaVersion(ctx)
|
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
||||||
|
schema, err := t.Database.SchemaVersion(callCtx)
|
||||||
|
cancel()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("read schema version: %w", err)
|
return false, fmt.Errorf("read schema version: %w", err)
|
||||||
}
|
}
|
||||||
@@ -319,7 +321,8 @@ func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, err
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// setImage retries while the State Store comes up.
|
// setImage retries while the State Store comes up. No attempt outlives the
|
||||||
|
// deadline, so a peer that never answers still ends in rollback.
|
||||||
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
|
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
|
||||||
if j.Managed {
|
if j.Managed {
|
||||||
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
|
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
|
||||||
@@ -328,7 +331,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
|
|||||||
}
|
}
|
||||||
deadline := time.Now().Add(t.HealthTimeout)
|
deadline := time.Now().Add(t.HealthTimeout)
|
||||||
for {
|
for {
|
||||||
err := t.Policy.SetImage(ctx, expected, desired)
|
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||||
|
err := t.Policy.SetImage(callCtx, expected, desired)
|
||||||
|
cancel()
|
||||||
if err == nil || time.Now().After(deadline) {
|
if err == nil || time.Now().After(deadline) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -341,7 +346,9 @@ func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired
|
|||||||
func (t *Transaction) observe(ctx context.Context, version string) error {
|
func (t *Transaction) observe(ctx context.Context, version string) error {
|
||||||
deadline := time.Now().Add(t.HealthTimeout)
|
deadline := time.Now().Add(t.HealthTimeout)
|
||||||
for {
|
for {
|
||||||
reported, err := t.Health.Healthy(ctx)
|
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
||||||
|
reported, err := t.Health.Healthy(callCtx)
|
||||||
|
cancel()
|
||||||
if err == nil && sameVersion(reported, version) {
|
if err == nil && sameVersion(reported, version) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -358,7 +365,9 @@ func (t *Transaction) observe(ctx context.Context, version string) error {
|
|||||||
if err := sleep(ctx, t.Poll); err != nil {
|
if err := sleep(ctx, t.Poll); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
reported, err := t.Health.Healthy(ctx)
|
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
||||||
|
reported, err := t.Health.Healthy(callCtx)
|
||||||
|
cancel()
|
||||||
if err != nil || !sameVersion(reported, version) {
|
if err != nil || !sameVersion(reported, version) {
|
||||||
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
|
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ type fakeHost struct {
|
|||||||
managed bool
|
managed bool
|
||||||
migrates bool
|
migrates bool
|
||||||
unhealthy bool
|
unhealthy bool
|
||||||
|
hangs bool // the State Store never answers the switch to the candidate
|
||||||
image string
|
image string
|
||||||
backedUp string
|
backedUp string
|
||||||
schema int64
|
schema int64
|
||||||
@@ -59,7 +60,11 @@ func (h *fakeHost) Pull(_ context.Context, ref string) error {
|
|||||||
|
|
||||||
func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil }
|
func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil }
|
||||||
|
|
||||||
func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error {
|
func (h *fakeHost) SetImage(ctx context.Context, expected, desired string) error {
|
||||||
|
if h.hangs && desired == newImage {
|
||||||
|
<-ctx.Done()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
if h.image != desired && h.image != expected {
|
if h.image != desired && h.image != expected {
|
||||||
return errors.New("image policy changed")
|
return errors.New("image policy changed")
|
||||||
}
|
}
|
||||||
@@ -147,6 +152,7 @@ func TestTransaction(t *testing.T) {
|
|||||||
{name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}},
|
{name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}},
|
||||||
{name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old},
|
{name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old},
|
||||||
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
|
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
|
||||||
|
{name: "stuck image switch rolls back", host: fakeHost{managed: true, hangs: true}, want: old},
|
||||||
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
|
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
|
||||||
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
|
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
|
||||||
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
|
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
|
||||||
@@ -171,8 +177,6 @@ func TestTransaction(t *testing.T) {
|
|||||||
for _, unit := range testUnits {
|
for _, unit := range testUnits {
|
||||||
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
|
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
|
||||||
}
|
}
|
||||||
self := filepath.Join(root, "self")
|
|
||||||
writeTestFile(t, self, "frozen watchdog")
|
|
||||||
report := filepath.Join(root, "report.json")
|
report := filepath.Join(root, "report.json")
|
||||||
tx := &Transaction{
|
tx := &Transaction{
|
||||||
StateDir: filepath.Join(root, "state"), BinDir: host.bin,
|
StateDir: filepath.Join(root, "state"), BinDir: host.bin,
|
||||||
@@ -180,7 +184,7 @@ func TestTransaction(t *testing.T) {
|
|||||||
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
|
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
|
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
|
||||||
switch {
|
switch {
|
||||||
case tt.prepare != "":
|
case tt.prepare != "":
|
||||||
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
|
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
|
||||||
@@ -188,11 +192,17 @@ func TestTransaction(t *testing.T) {
|
|||||||
}
|
}
|
||||||
case err != nil:
|
case err != nil:
|
||||||
t.Fatalf("Prepare: %v", err)
|
t.Fatalf("Prepare: %v", err)
|
||||||
|
case !strings.Contains(readTestFile(t, filepath.Join(JournalPath(tx.StateDir), WatchdogBinary)), " 2.0.0"):
|
||||||
|
t.Fatal("the frozen watchdog is not the release's archied")
|
||||||
case tt.crashAt != "":
|
case tt.crashAt != "":
|
||||||
crash(t, tx, &host, tt.crashAt)
|
crash(t, tx, &host, tt.crashAt)
|
||||||
err = tx.Run(t.Context(), true)
|
err = tx.Run(t.Context(), true)
|
||||||
default:
|
default:
|
||||||
err = tx.Run(t.Context(), false)
|
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err = tx.Run(ctx, false); ctx.Err() != nil {
|
||||||
|
t.Fatal("Run outlived HealthTimeout")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if tt.prepare == "" && err != nil {
|
if tt.prepare == "" && err != nil {
|
||||||
t.Fatalf("Run: %v", err)
|
t.Fatalf("Run: %v", err)
|
||||||
@@ -286,3 +296,12 @@ func writeTestFile(t *testing.T, path, data string) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func readTestFile(t *testing.T, path string) string {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user