fix(archied): recover an orphaned update journal while running, not only at boot

This commit is contained in:
sam committed 2026-10-27 15:24:04 +11:00
1 parent e5383b2129
commit d3fd50f6b0
2 files changed
+49 -4

No files matched your search

+2
View File
@@ -19,6 +19,7 @@ import (
"strconv"
"strings"
"syscall"
"time"
"github.com/moby/moby/client"
natsio "github.com/nats-io/nats.go"
@@ -173,6 +174,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
return 1
}
go b.applyStatus.Run(ctx)
go watchUpdateRecovery(ctx, args.cfgPath, 30*time.Second)
if exit, err := b.handleRequeue(ctx, args.requeue, args.once); err != nil {
return 1
} else if exit {
+47 -4
View File
@@ -299,6 +299,18 @@ func updateStateDir() string {
// reports that this boot must stop: recovery may replace this executable and
// restart this unit, so it runs outside it.
func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
orphaned, err := updateOrphaned()
if err != nil || !orphaned {
return false, err
}
if err := spawnRecovery(ctx, configPath); err != nil {
return false, err
}
return true, nil
}
// updateOrphaned reports a journal that no live watchdog holds the lock for.
func updateOrphaned() (bool, error) {
journal := releaseupdate.JournalPath(updateStateDir())
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
return false, nil
@@ -317,13 +329,44 @@ func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
return false, err
}
_ = unix.Flock(int(lock.Fd()), unix.LOCK_UN)
frozen := filepath.Join(journal, releaseupdate.WatchdogBinary)
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
return false, fmt.Errorf("schedule update recovery: %w", err)
}
return true, nil
}
func spawnRecovery(ctx context.Context, configPath string) error {
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
return fmt.Errorf("schedule update recovery: %w", err)
}
return nil
}
// watchUpdateRecovery recovers a transaction whose watchdog died while this
// daemon kept running. An orphan must be seen on two consecutive checks so a
// journal published just before its watchdog takes the lock is left alone.
func watchUpdateRecovery(ctx context.Context, configPath string, every time.Duration) {
ticker := time.NewTicker(every)
defer ticker.Stop()
seen := false
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
orphaned, err := updateOrphaned()
if err != nil {
slog.WarnContext(ctx, "check update recovery", "error", err)
continue
}
if orphaned && seen {
if err := spawnRecovery(ctx, configPath); err != nil {
slog.WarnContext(ctx, "recover update", "error", err)
}
}
seen = orphaned
}
}
func observedUnits(output []byte, err error) map[string]bool {
units := make(map[string]bool)
if err != nil {