package releaseupdate import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "slices" "time" "golang.org/x/sys/unix" ) // A transaction's journal moves through these phases. prepared means nothing // live has changed yet; committed and rolled-back mean the outcome is decided. // Any other phase found by a recovering watchdog means binaries or the image // policy may be half switched, and it rolls back. const ( phasePrepared = "prepared" phaseSwitching = "switching" phaseObserving = "observing" phaseRollingBack = "rolling-back" phaseCommitted = "committed" phaseRolledBack = "rolled-back" ) const ( // stateStoreUnit is started ahead of the others so the image policy can // be switched before archied reads it. stateStoreUnit = "archie-state-store" agentVersionFile = "archie-agent.version" // WatchdogBinary is the archied copy frozen into the journal: the binary // that wrote a journal is the one that interprets it, even after the // executables in the bin directory have been replaced. WatchdogBinary = "archied" journalDir = "transaction" journalFile = "journal.json" snapshotFile = "db.pre-update.dump" lockWait = 60 * time.Second ) // Units starts and stops the host's services. Managed is false where no // service manager runs them; then nothing is restarted. type Units interface { Managed(ctx context.Context) bool Start(ctx context.Context, unit string) error Stop(ctx context.Context, unit string) error } // Images fetches the agent image a release pins. type Images interface { Pull(ctx context.Context, ref string) error } // ImagePolicy is the control plane's agent image setting. type ImagePolicy interface { Image(ctx context.Context) (string, error) // SetImage replaces expected with desired. It succeeds without writing // when desired is already set, and refuses any other current value. SetImage(ctx context.Context, expected, desired string) error } // Database is the task database the services share. type Database interface { Backup(ctx context.Context, out string) error SchemaVersion(ctx context.Context) (int64, error) Restore(ctx context.Context, from string) error } // Health asks the daemon whether it is healthy. type Health interface { // Healthy returns the version a healthy daemon reports, or why it is not. Healthy(ctx context.Context) (string, error) } // Transaction installs a release and either commits it or rolls the host back // to the previous binaries and image policy. Prepare stages everything without // touching what runs; Run, in a process of its own, switches over, watches the // daemon and decides. type Transaction struct { StateDir string BinDir string Units Units Images Images Policy ImagePolicy Database Database Health Health Progress func(string) // HealthTimeout bounds the wait for the restarted daemon to report the // release; it must then stay healthy for Observe, checked every Poll. HealthTimeout time.Duration Observe time.Duration Poll time.Duration } type imageChange struct { Previous string `json:"previous"` Candidate string `json:"candidate"` } type journal struct { Phase string `json:"phase"` Version string `json:"version"` Previous map[string]string `json:"previous"` Units []string `json:"units"` Managed bool `json:"managed"` Image imageChange `json:"image"` SchemaBefore int64 `json:"schema_before"` Meta InstallMeta `json:"meta"` // RestoredDatabase records that rollback found a migration and restored // the pre-update snapshot. RestoredDatabase bool `json:"restored_database"` } // JournalPath is where an unfinished transaction lives. func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) } // Run drives the transaction in the journal to an outcome. recovering is set // on boot: it never waits for the lock, and a transaction that was prepared // but never started is abandoned rather than started unattended. func (t *Transaction) Run(ctx context.Context, recovering bool) error { unlock, held, err := lockState(ctx, t.StateDir, !recovering) if err != nil { return err } if !held { // A live installer or watchdog owns the journal. return nil } defer unlock() j, found, err := t.load() if err != nil || !found { return err } switch j.Phase { case phasePrepared: if recovering { return t.abandon(ctx, j) } return t.promote(ctx, j) case phaseCommitted, phaseRolledBack: // The outcome was decided before the watchdog died. archied exits at // boot while a journal exists, so the units come up again here. if err := t.startUnits(ctx, j, ""); err != nil { return err } return t.remove() default: return t.rollback(ctx, j) } } func (t *Transaction) promote(ctx context.Context, j journal) error { if err := t.setPhase(&j, phaseSwitching); err != nil { return err } if err := t.switchOver(ctx, &j); err != nil { t.say("update to %s failed: %v; rolling back", j.Version, err) return t.rollback(ctx, j) } if j.Managed { if err := t.writeReport(j, "passed", false); err != nil { return err } } else { t.say("installed %s; restart the archie services to run it", j.Version) } if err := t.setPhase(&j, phaseCommitted); err != nil { return err } return t.remove() } // switchOver replaces the executables and image policy, then restarts the // units in release order and requires the daemon to stay healthy. func (t *Transaction) switchOver(ctx context.Context, j *journal) error { if err := t.stopUnits(ctx, *j); err != nil { return err } for _, unit := range j.Units { if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil { return fmt.Errorf("switch %s: %w", unit, err) } } if err := writeFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n")); err != nil { return err } if err := syncDir(t.BinDir); err != nil { return err } if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil { return err } if !j.Managed { return nil } if err := t.setPhase(j, phaseObserving); err != nil { return err } if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil { return err } return t.observe(ctx, j.Version) } // rollback is safe to repeat from any point: every step converges on the // previous state whether or not an earlier attempt got that far. func (t *Transaction) rollback(ctx context.Context, j journal) error { if err := t.setPhase(&j, phaseRollingBack); err != nil { return err } if err := t.stopUnits(ctx, j); err != nil { return t.incomplete(err) } if err := t.restoreFiles(j); err != nil { return t.incomplete(err) } restored, err := t.restoreDatabase(ctx, j) if err != nil { return t.incomplete(err) } started := "" if restored { // The snapshot predates the policy switch, so it holds the previous image. j.RestoredDatabase = true if err := t.setPhase(&j, phaseRollingBack); err != nil { return t.incomplete(err) } t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot()) } else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil { return t.incomplete(err) } else { started = stateStoreUnit } if err := t.writeReport(j, "failed", true); err != nil { return t.incomplete(err) } if err := t.setPhase(&j, phaseRolledBack); err != nil { return t.incomplete(err) } if err := t.startUnits(ctx, j, started); err != nil { return t.incomplete(err) } return t.remove() } // abandon discards a transaction that never switched anything. func (t *Transaction) abandon(ctx context.Context, j journal) error { t.say("update to %s was staged but never started; discarding it", j.Version) for _, unit := range j.Units { if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } if err := t.writeReport(j, "failed", true); err != nil { return err } if err := t.setPhase(&j, phaseRolledBack); err != nil { return err } if err := t.startUnits(ctx, j, ""); err != nil { return err } return t.remove() } func (t *Transaction) incomplete(err error) error { return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err) } func (t *Transaction) restoreFiles(j journal) error { previous := filepath.Join(JournalPath(t.StateDir), "previous") for _, unit := range j.Units { live := filepath.Join(t.BinDir, unit) saved := filepath.Join(previous, unit) if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) { if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } else if err := installFile(saved, live, 0o755); err != nil { return fmt.Errorf("restore %s: %w", unit, err) } if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } version := filepath.Join(t.BinDir, agentVersionFile) saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile) if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) { if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } else if err := installFile(saved, version, 0o644); err != nil { return err } return syncDir(t.BinDir) } // restoreDatabase restores the pre-update snapshot only when the candidate // migrated the schema: the previous binaries refuse a newer schema, and data // written since the snapshot is otherwise worth keeping. func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) { if j.RestoredDatabase { return true, nil } callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout) schema, err := t.Database.SchemaVersion(callCtx) cancel() if err != nil { return false, fmt.Errorf("read schema version: %w", err) } if schema <= j.SchemaBefore { return false, nil } if err := t.Database.Restore(ctx, t.snapshot()); err != nil { return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err) } return true, nil } // setImage retries while the State Store comes up. No attempt outlives the // deadline, so a peer that never answers still ends in rollback. func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error { if j.Managed { if err := t.Units.Start(ctx, stateStoreUnit); err != nil { return err } } deadline := time.Now().Add(t.HealthTimeout) for { callCtx, cancel := context.WithDeadline(ctx, deadline) err := t.Policy.SetImage(callCtx, expected, desired) cancel() if err == nil || time.Now().After(deadline) { return err } if err := sleep(ctx, t.Poll); err != nil { return err } } } func (t *Transaction) observe(ctx context.Context, version string) error { deadline := time.Now().Add(t.HealthTimeout) for { callCtx, cancel := context.WithDeadline(ctx, deadline) reported, err := t.Health.Healthy(callCtx) cancel() if err == nil && sameVersion(reported, version) { break } if time.Now().After(deadline) { return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err)) } if err := sleep(ctx, t.Poll); err != nil { return err } } t.say("archied is healthy on %s; watching it for %s", version, t.Observe) until := time.Now().Add(t.Observe) for time.Now().Before(until) { if err := sleep(ctx, t.Poll); err != nil { return err } callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout) reported, err := t.Health.Healthy(callCtx) cancel() if err != nil || !sameVersion(reported, version) { return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err)) } } return nil } func healthProblem(reported string, err error) error { if err != nil { return err } return fmt.Errorf("it reports version %q", reported) } func sameVersion(left, right string) bool { l, ok := comparableVersion(left) r, _ := comparableVersion(right) return ok && l == r } // startUnits restarts the units in release order, skipping one already started. func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error { if !j.Managed { return nil } for _, unit := range j.Units { if unit == skip { continue } if err := t.Units.Start(ctx, unit); err != nil { return fmt.Errorf("start %s: %w", unit, err) } } return nil } // stopUnits stops in reverse release order, so callers go before the services // they depend on. func (t *Transaction) stopUnits(ctx context.Context, j journal) error { if !j.Managed { return nil } for _, unit := range slices.Backward(j.Units) { if err := t.Units.Stop(ctx, unit); err != nil { return fmt.Errorf("stop %s: %w", unit, err) } } return nil } func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error { if j.Meta.ReportPath == "" { return nil } installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version} return WritePendingReport(j.Meta.ReportPath, Report{ Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID, Previous: j.Previous, Installed: installed, HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase, }) } func (t *Transaction) staged(unit string) string { return filepath.Join(t.BinDir, "."+unit+".new") } func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) } func (t *Transaction) say(format string, args ...any) { if t.Progress != nil { t.Progress(fmt.Sprintf(format, args...)) } } func (t *Transaction) load() (journal, bool, error) { data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile)) if errors.Is(err, os.ErrNotExist) { return journal{}, false, nil } if err != nil { return journal{}, false, fmt.Errorf("read update journal: %w", err) } var j journal if err := json.Unmarshal(data, &j); err != nil { return journal{}, false, fmt.Errorf("decode update journal: %w", err) } return j, true, nil } func (t *Transaction) setPhase(j *journal, phase string) error { j.Phase = phase return writeJournal(JournalPath(t.StateDir), *j) } func writeJournal(dir string, j journal) error { data, err := json.Marshal(j) if err != nil { return err } if err := writeFileAtomic(filepath.Join(dir, journalFile), data); err != nil { return err } return syncDir(dir) } // remove retires the journal by rename first, so a crash mid-delete never // leaves a partial journal behind. func (t *Transaction) remove() error { finished := filepath.Join(t.StateDir, "finished") if err := os.RemoveAll(finished); err != nil { return err } if err := os.Rename(JournalPath(t.StateDir), finished); err != nil { return err } if err := syncDir(t.StateDir); err != nil { return err } return os.RemoveAll(finished) } // lockState takes the update lock. Without wait it reports held=false at once // when another process owns it. func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) { if err := os.MkdirAll(dir, 0o700); err != nil { return nil, false, err } file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return nil, false, err } deadline := time.Now().Add(lockWait) for { err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) if err == nil { return func() { _ = file.Close() }, true, nil } if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) { _ = file.Close() if errors.Is(err, unix.EWOULDBLOCK) { return nil, false, nil } return nil, false, err } if err := sleep(ctx, 100*time.Millisecond); err != nil { _ = file.Close() return nil, false, err } } } func sleep(ctx context.Context, d time.Duration) error { timer := time.NewTimer(d) defer timer.Stop() select { case <-ctx.Done(): return ctx.Err() case <-timer.C: return nil } } func syncDir(dir string) error { file, err := os.Open(dir) if err != nil { return err } defer func() { _ = file.Close() }() return file.Sync() } // installFile copies src over dst through a synced temp file and rename, so // dst is always either the old or the new content. func installFile(src, dst string, mode os.FileMode) error { data, err := os.ReadFile(src) if err != nil { return err } tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp") if err := writeSynced(tmp, data, mode); err != nil { return err } return os.Rename(tmp, dst) } func writeSynced(path string, data []byte, mode os.FileMode) error { file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) if err != nil { return err } if _, err := file.Write(data); err != nil { _ = file.Close() return err } if err := file.Chmod(mode); err != nil { _ = file.Close() return err } if err := file.Sync(); err != nil { _ = file.Close() return err } return file.Close() }