fix(archied): recover an orphaned update journal while running, not only at boot
This commit is contained in:
1 parent
e5383b2129
commit
d3fd50f6b0
2 files changed
+49
-4
No files matched your search
@@ -19,6 +19,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
@@ -173,6 +174,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
return 1
|
||||
}
|
||||
go b.applyStatus.Run(ctx)
|
||||
go watchUpdateRecovery(ctx, args.cfgPath, 30*time.Second)
|
||||
if exit, err := b.handleRequeue(ctx, args.requeue, args.once); err != nil {
|
||||
return 1
|
||||
} else if exit {
|
||||
|
||||
@@ -299,6 +299,18 @@ func updateStateDir() string {
|
||||
// reports that this boot must stop: recovery may replace this executable and
|
||||
// restart this unit, so it runs outside it.
|
||||
func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
|
||||
orphaned, err := updateOrphaned()
|
||||
if err != nil || !orphaned {
|
||||
return false, err
|
||||
}
|
||||
if err := spawnRecovery(ctx, configPath); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// updateOrphaned reports a journal that no live watchdog holds the lock for.
|
||||
func updateOrphaned() (bool, error) {
|
||||
journal := releaseupdate.JournalPath(updateStateDir())
|
||||
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
@@ -317,13 +329,44 @@ func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
|
||||
return false, err
|
||||
}
|
||||
_ = unix.Flock(int(lock.Fd()), unix.LOCK_UN)
|
||||
frozen := filepath.Join(journal, releaseupdate.WatchdogBinary)
|
||||
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
|
||||
return false, fmt.Errorf("schedule update recovery: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func spawnRecovery(ctx context.Context, configPath string) error {
|
||||
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
|
||||
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
|
||||
return fmt.Errorf("schedule update recovery: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchUpdateRecovery recovers a transaction whose watchdog died while this
|
||||
// daemon kept running. An orphan must be seen on two consecutive checks so a
|
||||
// journal published just before its watchdog takes the lock is left alone.
|
||||
func watchUpdateRecovery(ctx context.Context, configPath string, every time.Duration) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
seen := false
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
orphaned, err := updateOrphaned()
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "check update recovery", "error", err)
|
||||
continue
|
||||
}
|
||||
if orphaned && seen {
|
||||
if err := spawnRecovery(ctx, configPath); err != nil {
|
||||
slog.WarnContext(ctx, "recover update", "error", err)
|
||||
}
|
||||
}
|
||||
seen = orphaned
|
||||
}
|
||||
}
|
||||
|
||||
func observedUnits(output []byte, err error) map[string]bool {
|
||||
units := make(map[string]bool)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user