From abe4c488cf6c26bce98374d8446c78f54cfb4273 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 27 Oct 2026 15:23:51 +1100 Subject: [PATCH] fix(archied): recover an orphaned update journal while running, not only at boot --- internal/app/archied/main.go | 2 ++ internal/app/archied/update.go | 51 +++++++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 90d203fd5..766d6ab15 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -19,6 +19,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/moby/moby/client" natsio "github.com/nats-io/nats.go" @@ -173,6 +174,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence return 1 } go b.applyStatus.Run(ctx) + go watchUpdateRecovery(ctx, args.cfgPath, 30*time.Second) if exit, err := b.handleRequeue(ctx, args.requeue, args.once); err != nil { return 1 } else if exit { diff --git a/internal/app/archied/update.go b/internal/app/archied/update.go index 489ab2b76..66b920f14 100644 --- a/internal/app/archied/update.go +++ b/internal/app/archied/update.go @@ -299,6 +299,18 @@ func updateStateDir() string { // reports that this boot must stop: recovery may replace this executable and // restart this unit, so it runs outside it. func resumeUpdate(ctx context.Context, configPath string) (bool, error) { + orphaned, err := updateOrphaned() + if err != nil || !orphaned { + return false, err + } + if err := spawnRecovery(ctx, configPath); err != nil { + return false, err + } + return true, nil +} + +// updateOrphaned reports a journal that no live watchdog holds the lock for. +func updateOrphaned() (bool, error) { journal := releaseupdate.JournalPath(updateStateDir()) if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) { return false, nil @@ -317,13 +329,44 @@ func resumeUpdate(ctx context.Context, configPath string) (bool, error) { return false, err } _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) - frozen := filepath.Join(journal, releaseupdate.WatchdogBinary) - if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil { - return false, fmt.Errorf("schedule update recovery: %w", err) - } return true, nil } +func spawnRecovery(ctx context.Context, configPath string) error { + frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary) + if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil { + return fmt.Errorf("schedule update recovery: %w", err) + } + return nil +} + +// watchUpdateRecovery recovers a transaction whose watchdog died while this +// daemon kept running. An orphan must be seen on two consecutive checks so a +// journal published just before its watchdog takes the lock is left alone. +func watchUpdateRecovery(ctx context.Context, configPath string, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + seen := false + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + orphaned, err := updateOrphaned() + if err != nil { + slog.WarnContext(ctx, "check update recovery", "error", err) + continue + } + if orphaned && seen { + if err := spawnRecovery(ctx, configPath); err != nil { + slog.WarnContext(ctx, "recover update", "error", err) + } + } + seen = orphaned + } +} + func observedUnits(output []byte, err error) map[string]bool { units := make(map[string]bool) if err != nil {