Files
archie/internal/releaseupdate/transaction.go
T

576 lines
17 KiB
Go

package releaseupdate
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"time"
"golang.org/x/sys/unix"
)
// A transaction's journal moves through these phases. prepared means nothing
// live has changed yet; committed and rolled-back mean the outcome is decided.
// Any other phase found by a recovering watchdog means binaries or the image
// policy may be half switched, and it rolls back.
const (
phasePrepared = "prepared"
phaseSwitching = "switching"
phaseObserving = "observing"
phaseRollingBack = "rolling-back"
phaseCommitted = "committed"
phaseRolledBack = "rolled-back"
)
const (
// stateStoreUnit is started ahead of the others so the image policy can
// be switched before archied reads it.
stateStoreUnit = "archie-state-store"
agentVersionFile = "archie-agent.version"
// WatchdogBinary is the archied copy frozen into the journal: the binary
// that wrote a journal is the one that interprets it, even after the
// executables in the bin directory have been replaced.
WatchdogBinary = "archied"
journalDir = "transaction"
journalFile = "journal.json"
snapshotFile = "db.pre-update.dump"
lockWait = 60 * time.Second
)
// Units starts and stops the host's services. Managed is false where no
// service manager runs them; then nothing is restarted.
type Units interface {
Managed(ctx context.Context) bool
Start(ctx context.Context, unit string) error
Stop(ctx context.Context, unit string) error
}
// Images fetches the agent image a release pins.
type Images interface {
Pull(ctx context.Context, ref string) error
}
// ImagePolicy is the control plane's agent image setting.
type ImagePolicy interface {
Image(ctx context.Context) (string, error)
// SetImage replaces expected with desired. It succeeds without writing
// when desired is already set, and refuses any other current value.
SetImage(ctx context.Context, expected, desired string) error
}
// Database is the task database the services share.
type Database interface {
Backup(ctx context.Context, out string) error
SchemaVersion(ctx context.Context) (int64, error)
Restore(ctx context.Context, from string) error
}
// Health asks the daemon whether it is healthy.
type Health interface {
// Healthy returns the version a healthy daemon reports, or why it is not.
Healthy(ctx context.Context) (string, error)
}
// Transaction installs a release and either commits it or rolls the host back
// to the previous binaries and image policy. Prepare stages everything without
// touching what runs; Run, in a process of its own, switches over, watches the
// daemon and decides.
type Transaction struct {
StateDir string
BinDir string
Units Units
Images Images
Policy ImagePolicy
Database Database
Health Health
Progress func(string)
// HealthTimeout bounds the wait for the restarted daemon to report the
// release; it must then stay healthy for Observe, checked every Poll.
HealthTimeout time.Duration
Observe time.Duration
Poll time.Duration
}
type imageChange struct {
Previous string `json:"previous"`
Candidate string `json:"candidate"`
}
type journal struct {
Phase string `json:"phase"`
Version string `json:"version"`
Previous map[string]string `json:"previous"`
Units []string `json:"units"`
Binaries []string `json:"binaries,omitempty"`
Managed bool `json:"managed"`
Image imageChange `json:"image"`
SchemaBefore int64 `json:"schema_before"`
Meta InstallMeta `json:"meta"`
// RestoredDatabase records that rollback found a migration and restored
// the pre-update snapshot.
RestoredDatabase bool `json:"restored_database"`
}
// JournalPath is where an unfinished transaction lives.
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
// Run drives the transaction in the journal to an outcome. recovering is set
// on boot: it never waits for the lock, and a transaction that was prepared
// but never started is abandoned rather than started unattended.
func (t *Transaction) Run(ctx context.Context, recovering bool) error {
unlock, held, err := lockState(ctx, t.StateDir, !recovering)
if err != nil {
return err
}
if !held {
// A live installer or watchdog owns the journal.
return nil
}
defer unlock()
j, found, err := t.load()
if err != nil || !found {
return err
}
switch j.Phase {
case phasePrepared:
if recovering {
return t.abandon(ctx, j)
}
return t.promote(ctx, j)
case phaseCommitted, phaseRolledBack:
// The outcome was decided before the watchdog died. archied exits at
// boot while a journal exists, so the units come up again here.
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
default:
return t.rollback(ctx, j)
}
}
func (t *Transaction) promote(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseSwitching); err != nil {
return err
}
if err := t.switchOver(ctx, &j); err != nil {
t.say("update to %s failed: %v; rolling back", j.Version, err)
return t.rollback(ctx, j)
}
if j.Managed {
if err := t.writeReport(j, "passed", false); err != nil {
return err
}
} else {
t.say("installed %s; restart the archie services to run it", j.Version)
}
if err := t.setPhase(&j, phaseCommitted); err != nil {
return err
}
return t.remove()
}
// switchOver replaces the executables and image policy, then restarts the
// units in release order and requires the daemon to stay healthy.
func (t *Transaction) switchOver(ctx context.Context, j *journal) error {
if err := t.stopUnits(ctx, *j); err != nil {
return err
}
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil {
return fmt.Errorf("switch %s: %w", unit, err)
}
}
if err := writeFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n")); err != nil {
return err
}
if err := syncDir(t.BinDir); err != nil {
return err
}
if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil {
return err
}
if !j.Managed {
return nil
}
if err := t.setPhase(j, phaseObserving); err != nil {
return err
}
if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil {
return err
}
return t.observe(ctx, j.Version)
}
// rollback is safe to repeat from any point: every step converges on the
// previous state whether or not an earlier attempt got that far.
func (t *Transaction) rollback(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return err
}
if err := t.stopUnits(ctx, j); err != nil {
return t.incomplete(err)
}
if err := t.restoreFiles(j); err != nil {
return t.incomplete(err)
}
restored, err := t.restoreDatabase(ctx, j)
if err != nil {
return t.incomplete(err)
}
started := ""
if restored {
// The snapshot predates the policy switch, so it holds the previous image.
j.RestoredDatabase = true
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return t.incomplete(err)
}
t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot())
} else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil {
return t.incomplete(err)
} else {
started = stateStoreUnit
}
if err := t.writeReport(j, "failed", true); err != nil {
return t.incomplete(err)
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return t.incomplete(err)
}
if err := t.startUnits(ctx, j, started); err != nil {
return t.incomplete(err)
}
return t.remove()
}
// abandon discards a transaction that never switched anything.
func (t *Transaction) abandon(ctx context.Context, j journal) error {
t.say("update to %s was staged but never started; discarding it", j.Version)
for _, app := range append(slices.Clone(j.Units), j.Binaries...) {
if err := os.Remove(t.staged(app)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
if err := t.writeReport(j, "failed", true); err != nil {
return err
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return err
}
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
}
func (t *Transaction) incomplete(err error) error {
return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err)
}
func (t *Transaction) restoreFiles(j journal) error {
previous := filepath.Join(JournalPath(t.StateDir), "previous")
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
live := filepath.Join(t.BinDir, unit)
saved := filepath.Join(previous, unit)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, live, 0o755); err != nil {
return fmt.Errorf("restore %s: %w", unit, err)
}
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
version := filepath.Join(t.BinDir, agentVersionFile)
saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, version, 0o644); err != nil {
return err
}
return syncDir(t.BinDir)
}
// restoreDatabase restores the pre-update snapshot only when the candidate
// migrated the schema: the previous binaries refuse a newer schema, and data
// written since the snapshot is otherwise worth keeping.
func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) {
if j.RestoredDatabase {
return true, nil
}
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
schema, err := t.Database.SchemaVersion(callCtx)
cancel()
if err != nil {
return false, fmt.Errorf("read schema version: %w", err)
}
if schema <= j.SchemaBefore {
return false, nil
}
if err := t.Database.Restore(ctx, t.snapshot()); err != nil {
return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err)
}
return true, nil
}
// setImage retries while the State Store comes up. No attempt outlives the
// deadline, so a peer that never answers still ends in rollback.
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
if j.Managed {
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
return err
}
}
deadline := time.Now().Add(t.HealthTimeout)
for {
callCtx, cancel := context.WithDeadline(ctx, deadline)
err := t.Policy.SetImage(callCtx, expected, desired)
cancel()
if err == nil || time.Now().After(deadline) {
return err
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
}
func (t *Transaction) observe(ctx context.Context, version string) error {
deadline := time.Now().Add(t.HealthTimeout)
for {
callCtx, cancel := context.WithDeadline(ctx, deadline)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err == nil && sameVersion(reported, version) {
break
}
if time.Now().After(deadline) {
return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err))
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
t.say("archied is healthy on %s; watching it for %s", version, t.Observe)
until := time.Now().Add(t.Observe)
for time.Now().Before(until) {
if err := sleep(ctx, t.Poll); err != nil {
return err
}
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err != nil || !sameVersion(reported, version) {
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
}
}
return nil
}
func healthProblem(reported string, err error) error {
if err != nil {
return err
}
return fmt.Errorf("it reports version %q", reported)
}
func sameVersion(left, right string) bool {
l, ok := comparableVersion(left)
r, _ := comparableVersion(right)
return ok && l == r
}
// startUnits restarts the units in release order, skipping one already started.
func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error {
if !j.Managed {
return nil
}
for _, unit := range j.Units {
if unit == skip {
continue
}
if err := t.Units.Start(ctx, unit); err != nil {
return fmt.Errorf("start %s: %w", unit, err)
}
}
return nil
}
// stopUnits stops in reverse release order, so callers go before the services
// they depend on.
func (t *Transaction) stopUnits(ctx context.Context, j journal) error {
if !j.Managed {
return nil
}
for _, unit := range slices.Backward(j.Units) {
if err := t.Units.Stop(ctx, unit); err != nil {
return fmt.Errorf("stop %s: %w", unit, err)
}
}
return nil
}
func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error {
if j.Meta.ReportPath == "" {
return nil
}
installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version}
return WritePendingReport(j.Meta.ReportPath, Report{
Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID,
Previous: j.Previous, Installed: installed,
HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase,
})
}
func (t *Transaction) staged(unit string) string {
return filepath.Join(t.BinDir, "."+unit+".new")
}
func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) }
func (t *Transaction) say(format string, args ...any) {
if t.Progress != nil {
t.Progress(fmt.Sprintf(format, args...))
}
}
func (t *Transaction) load() (journal, bool, error) {
data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile))
if errors.Is(err, os.ErrNotExist) {
return journal{}, false, nil
}
if err != nil {
return journal{}, false, fmt.Errorf("read update journal: %w", err)
}
var j journal
if err := json.Unmarshal(data, &j); err != nil {
return journal{}, false, fmt.Errorf("decode update journal: %w", err)
}
return j, true, nil
}
func (t *Transaction) setPhase(j *journal, phase string) error {
j.Phase = phase
return writeJournal(JournalPath(t.StateDir), *j)
}
func writeJournal(dir string, j journal) error {
data, err := json.Marshal(j)
if err != nil {
return err
}
if err := writeFileAtomic(filepath.Join(dir, journalFile), data); err != nil {
return err
}
return syncDir(dir)
}
// remove retires the journal by rename first, so a crash mid-delete never
// leaves a partial journal behind.
func (t *Transaction) remove() error {
finished := filepath.Join(t.StateDir, "finished")
if err := os.RemoveAll(finished); err != nil {
return err
}
if err := os.Rename(JournalPath(t.StateDir), finished); err != nil {
return err
}
if err := syncDir(t.StateDir); err != nil {
return err
}
return os.RemoveAll(finished)
}
// lockState takes the update lock. Without wait it reports held=false at once
// when another process owns it.
func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, false, err
}
file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, false, err
}
deadline := time.Now().Add(lockWait)
for {
err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err == nil {
return func() { _ = file.Close() }, true, nil
}
if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) {
_ = file.Close()
if errors.Is(err, unix.EWOULDBLOCK) {
return nil, false, nil
}
return nil, false, err
}
if err := sleep(ctx, 100*time.Millisecond); err != nil {
_ = file.Close()
return nil, false, err
}
}
}
func sleep(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func syncDir(dir string) error {
file, err := os.Open(dir)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
return file.Sync()
}
// installFile copies src over dst through a synced temp file and rename, so
// dst is always either the old or the new content.
func installFile(src, dst string, mode os.FileMode) error {
data, err := os.ReadFile(src)
if err != nil {
return err
}
tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp")
if err := writeSynced(tmp, data, mode); err != nil {
return err
}
return os.Rename(tmp, dst)
}
func writeSynced(path string, data []byte, mode os.FileMode) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return err
}
if _, err := file.Write(data); err != nil {
_ = file.Close()
return err
}
if err := file.Chmod(mode); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}