Files
archie/internal/releaseupdate/transaction.go
T
sam 417d379ede feat(releaseupdate): transactional self-update through archied (#1633)
archied update install verifies the release zip, checks every binary
reports the release, pulls the agent image by digest, backs up the
database in-process and stages the binaries before handing off to
archied update watchdog in its own process. The watchdog restarts units
in release order, requires archied healthy for 120s and otherwise rolls
back binaries and image policy, restoring the database only when a
migration ran. Boot recovery resumes the frozen watchdog.

Deletes scripts/archie-update-install, scripts/archie-update-watchdog,
the updater's --from-source and --auto, CommandInstaller,
ScriptInstaller and the update-image and update-topology subcommands.
2026-10-12 03:21:02 +11:00

566 lines
16 KiB
Go

package releaseupdate
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"time"
"golang.org/x/sys/unix"
)
// A transaction's journal moves through these phases. prepared means nothing
// live has changed yet; committed and rolled-back mean the outcome is decided.
// Any other phase found by a recovering watchdog means binaries or the image
// policy may be half switched, and it rolls back.
const (
phasePrepared = "prepared"
phaseSwitching = "switching"
phaseObserving = "observing"
phaseRollingBack = "rolling-back"
phaseCommitted = "committed"
phaseRolledBack = "rolled-back"
)
const (
// stateStoreUnit is started ahead of the others so the image policy can
// be switched before archied reads it.
stateStoreUnit = "archie-state-store"
agentVersionFile = "archie-agent.version"
// WatchdogBinary is the archied copy frozen into the journal: the binary
// that wrote a journal is the one that interprets it, even after the
// executables in the bin directory have been replaced.
WatchdogBinary = "archied"
journalDir = "transaction"
journalFile = "journal.json"
snapshotFile = "db.pre-update.dump"
lockWait = 60 * time.Second
)
// Units starts and stops the host's services. Managed is false where no
// service manager runs them; then nothing is restarted.
type Units interface {
Managed(ctx context.Context) bool
Start(ctx context.Context, unit string) error
Stop(ctx context.Context, unit string) error
}
// Images fetches the agent image a release pins.
type Images interface {
Pull(ctx context.Context, ref string) error
}
// ImagePolicy is the control plane's agent image setting.
type ImagePolicy interface {
Image(ctx context.Context) (string, error)
// SetImage replaces expected with desired. It succeeds without writing
// when desired is already set, and refuses any other current value.
SetImage(ctx context.Context, expected, desired string) error
}
// Database is the task database the services share.
type Database interface {
Backup(ctx context.Context, out string) error
SchemaVersion(ctx context.Context) (int64, error)
Restore(ctx context.Context, from string) error
}
// Health asks the daemon whether it is healthy.
type Health interface {
// Healthy returns the version a healthy daemon reports, or why it is not.
Healthy(ctx context.Context) (string, error)
}
// Transaction installs a release and either commits it or rolls the host back
// to the previous binaries and image policy. Prepare stages everything without
// touching what runs; Run, in a process of its own, switches over, watches the
// daemon and decides.
type Transaction struct {
StateDir string
BinDir string
Units Units
Images Images
Policy ImagePolicy
Database Database
Health Health
Progress func(string)
// HealthTimeout bounds the wait for the restarted daemon to report the
// release; it must then stay healthy for Observe, checked every Poll.
HealthTimeout time.Duration
Observe time.Duration
Poll time.Duration
}
type imageChange struct {
Previous string `json:"previous"`
Candidate string `json:"candidate"`
}
type journal struct {
Phase string `json:"phase"`
Version string `json:"version"`
Previous map[string]string `json:"previous"`
Units []string `json:"units"`
Managed bool `json:"managed"`
Image imageChange `json:"image"`
SchemaBefore int64 `json:"schema_before"`
Meta InstallMeta `json:"meta"`
// RestoredDatabase records that rollback found a migration and restored
// the pre-update snapshot.
RestoredDatabase bool `json:"restored_database"`
}
// JournalPath is where an unfinished transaction lives.
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
// Run drives the transaction in the journal to an outcome. recovering is set
// on boot: it never waits for the lock, and a transaction that was prepared
// but never started is abandoned rather than started unattended.
func (t *Transaction) Run(ctx context.Context, recovering bool) error {
unlock, held, err := lockState(ctx, t.StateDir, !recovering)
if err != nil {
return err
}
if !held {
// A live installer or watchdog owns the journal.
return nil
}
defer unlock()
j, found, err := t.load()
if err != nil || !found {
return err
}
switch j.Phase {
case phasePrepared:
if recovering {
return t.abandon(ctx, j)
}
return t.promote(ctx, j)
case phaseCommitted, phaseRolledBack:
// The outcome was decided before the watchdog died. archied exits at
// boot while a journal exists, so the units come up again here.
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
default:
return t.rollback(ctx, j)
}
}
func (t *Transaction) promote(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseSwitching); err != nil {
return err
}
if err := t.switchOver(ctx, &j); err != nil {
t.say("update to %s failed: %v; rolling back", j.Version, err)
return t.rollback(ctx, j)
}
if j.Managed {
if err := t.writeReport(j, "passed", false); err != nil {
return err
}
} else {
t.say("installed %s; restart the archie services to run it", j.Version)
}
if err := t.setPhase(&j, phaseCommitted); err != nil {
return err
}
return t.remove()
}
// switchOver replaces the executables and image policy, then restarts the
// units in release order and requires the daemon to stay healthy.
func (t *Transaction) switchOver(ctx context.Context, j *journal) error {
if err := t.stopUnits(ctx, *j); err != nil {
return err
}
for _, unit := range j.Units {
if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil {
return fmt.Errorf("switch %s: %w", unit, err)
}
}
if err := writeFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n")); err != nil {
return err
}
if err := syncDir(t.BinDir); err != nil {
return err
}
if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil {
return err
}
if !j.Managed {
return nil
}
if err := t.setPhase(j, phaseObserving); err != nil {
return err
}
if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil {
return err
}
return t.observe(ctx, j.Version)
}
// rollback is safe to repeat from any point: every step converges on the
// previous state whether or not an earlier attempt got that far.
func (t *Transaction) rollback(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return err
}
if err := t.stopUnits(ctx, j); err != nil {
return t.incomplete(err)
}
if err := t.restoreFiles(j); err != nil {
return t.incomplete(err)
}
restored, err := t.restoreDatabase(ctx, j)
if err != nil {
return t.incomplete(err)
}
started := ""
if restored {
// The snapshot predates the policy switch, so it holds the previous image.
j.RestoredDatabase = true
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return t.incomplete(err)
}
t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot())
} else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil {
return t.incomplete(err)
} else {
started = stateStoreUnit
}
if err := t.writeReport(j, "failed", true); err != nil {
return t.incomplete(err)
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return t.incomplete(err)
}
if err := t.startUnits(ctx, j, started); err != nil {
return t.incomplete(err)
}
return t.remove()
}
// abandon discards a transaction that never switched anything.
func (t *Transaction) abandon(ctx context.Context, j journal) error {
t.say("update to %s was staged but never started; discarding it", j.Version)
for _, unit := range j.Units {
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
if err := t.writeReport(j, "failed", true); err != nil {
return err
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return err
}
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
}
func (t *Transaction) incomplete(err error) error {
return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err)
}
func (t *Transaction) restoreFiles(j journal) error {
previous := filepath.Join(JournalPath(t.StateDir), "previous")
for _, unit := range j.Units {
live := filepath.Join(t.BinDir, unit)
saved := filepath.Join(previous, unit)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, live, 0o755); err != nil {
return fmt.Errorf("restore %s: %w", unit, err)
}
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
version := filepath.Join(t.BinDir, agentVersionFile)
saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, version, 0o644); err != nil {
return err
}
return syncDir(t.BinDir)
}
// restoreDatabase restores the pre-update snapshot only when the candidate
// migrated the schema: the previous binaries refuse a newer schema, and data
// written since the snapshot is otherwise worth keeping.
func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) {
if j.RestoredDatabase {
return true, nil
}
schema, err := t.Database.SchemaVersion(ctx)
if err != nil {
return false, fmt.Errorf("read schema version: %w", err)
}
if schema <= j.SchemaBefore {
return false, nil
}
if err := t.Database.Restore(ctx, t.snapshot()); err != nil {
return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err)
}
return true, nil
}
// setImage retries while the State Store comes up.
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
if j.Managed {
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
return err
}
}
deadline := time.Now().Add(t.HealthTimeout)
for {
err := t.Policy.SetImage(ctx, expected, desired)
if err == nil || time.Now().After(deadline) {
return err
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
}
func (t *Transaction) observe(ctx context.Context, version string) error {
deadline := time.Now().Add(t.HealthTimeout)
for {
reported, err := t.Health.Healthy(ctx)
if err == nil && sameVersion(reported, version) {
break
}
if time.Now().After(deadline) {
return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err))
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
t.say("archied is healthy on %s; watching it for %s", version, t.Observe)
until := time.Now().Add(t.Observe)
for time.Now().Before(until) {
if err := sleep(ctx, t.Poll); err != nil {
return err
}
reported, err := t.Health.Healthy(ctx)
if err != nil || !sameVersion(reported, version) {
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
}
}
return nil
}
func healthProblem(reported string, err error) error {
if err != nil {
return err
}
return fmt.Errorf("it reports version %q", reported)
}
func sameVersion(left, right string) bool {
l, ok := comparableVersion(left)
r, _ := comparableVersion(right)
return ok && l == r
}
// startUnits restarts the units in release order, skipping one already started.
func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error {
if !j.Managed {
return nil
}
for _, unit := range j.Units {
if unit == skip {
continue
}
if err := t.Units.Start(ctx, unit); err != nil {
return fmt.Errorf("start %s: %w", unit, err)
}
}
return nil
}
// stopUnits stops in reverse release order, so callers go before the services
// they depend on.
func (t *Transaction) stopUnits(ctx context.Context, j journal) error {
if !j.Managed {
return nil
}
for _, unit := range slices.Backward(j.Units) {
if err := t.Units.Stop(ctx, unit); err != nil {
return fmt.Errorf("stop %s: %w", unit, err)
}
}
return nil
}
func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error {
if j.Meta.ReportPath == "" {
return nil
}
installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version}
return WritePendingReport(j.Meta.ReportPath, Report{
Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID,
Previous: j.Previous, Installed: installed,
HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase,
})
}
func (t *Transaction) staged(unit string) string {
return filepath.Join(t.BinDir, "."+unit+".new")
}
func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) }
func (t *Transaction) say(format string, args ...any) {
if t.Progress != nil {
t.Progress(fmt.Sprintf(format, args...))
}
}
func (t *Transaction) load() (journal, bool, error) {
data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile))
if errors.Is(err, os.ErrNotExist) {
return journal{}, false, nil
}
if err != nil {
return journal{}, false, fmt.Errorf("read update journal: %w", err)
}
var j journal
if err := json.Unmarshal(data, &j); err != nil {
return journal{}, false, fmt.Errorf("decode update journal: %w", err)
}
return j, true, nil
}
func (t *Transaction) setPhase(j *journal, phase string) error {
j.Phase = phase
return writeJournal(JournalPath(t.StateDir), *j)
}
func writeJournal(dir string, j journal) error {
data, err := json.Marshal(j)
if err != nil {
return err
}
if err := writeFileAtomic(filepath.Join(dir, journalFile), data); err != nil {
return err
}
return syncDir(dir)
}
// remove retires the journal by rename first, so a crash mid-delete never
// leaves a partial journal behind.
func (t *Transaction) remove() error {
finished := filepath.Join(t.StateDir, "finished")
if err := os.RemoveAll(finished); err != nil {
return err
}
if err := os.Rename(JournalPath(t.StateDir), finished); err != nil {
return err
}
if err := syncDir(t.StateDir); err != nil {
return err
}
return os.RemoveAll(finished)
}
// lockState takes the update lock. Without wait it reports held=false at once
// when another process owns it.
func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, false, err
}
file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, false, err
}
deadline := time.Now().Add(lockWait)
for {
err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err == nil {
return func() { _ = file.Close() }, true, nil
}
if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) {
_ = file.Close()
if errors.Is(err, unix.EWOULDBLOCK) {
return nil, false, nil
}
return nil, false, err
}
if err := sleep(ctx, 100*time.Millisecond); err != nil {
_ = file.Close()
return nil, false, err
}
}
}
func sleep(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func syncDir(dir string) error {
file, err := os.Open(dir)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
return file.Sync()
}
// installFile copies src over dst through a synced temp file and rename, so
// dst is always either the old or the new content.
func installFile(src, dst string, mode os.FileMode) error {
data, err := os.ReadFile(src)
if err != nil {
return err
}
tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp")
if err := writeSynced(tmp, data, mode); err != nil {
return err
}
return os.Rename(tmp, dst)
}
func writeSynced(path string, data []byte, mode os.FileMode) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return err
}
if _, err := file.Write(data); err != nil {
_ = file.Close()
return err
}
if err := file.Chmod(mode); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}