586 lines
17 KiB
Go
586 lines
17 KiB
Go
package releaseupdate
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"time"
|
|
|
|
"golang.org/x/sys/unix"
|
|
|
|
"github.com/samcharles93/archie-core/internal/fsutil"
|
|
)
|
|
|
|
// A transaction's journal moves through these phases. prepared means nothing
|
|
// live has changed yet; committed and rolled-back mean the outcome is decided.
|
|
// Any other phase found by a recovering watchdog means binaries or the image
|
|
// policy may be half switched, and it rolls back.
|
|
const (
|
|
phasePrepared = "prepared"
|
|
phaseSwitching = "switching"
|
|
phaseObserving = "observing"
|
|
phaseRollingBack = "rolling-back"
|
|
phaseCommitted = "committed"
|
|
phaseRolledBack = "rolled-back"
|
|
)
|
|
|
|
const (
|
|
// stateStoreUnit is started ahead of the others so the image policy can
|
|
// be switched before archied reads it.
|
|
stateStoreUnit = "archie-state-store"
|
|
agentVersionFile = "archie-agent.version"
|
|
// WatchdogBinary is the archied copy frozen into the journal: the binary
|
|
// that wrote a journal is the one that interprets it, even after the
|
|
// executables in the bin directory have been replaced.
|
|
WatchdogBinary = "archied"
|
|
journalDir = "transaction"
|
|
journalFile = "journal.json"
|
|
snapshotFile = "db.pre-update.dump"
|
|
lockWait = 60 * time.Second
|
|
)
|
|
|
|
// Units starts and stops the host's services. Managed is false where no
|
|
// service manager runs them; then nothing is restarted.
|
|
type Units interface {
|
|
Managed(ctx context.Context) bool
|
|
Start(ctx context.Context, unit string) error
|
|
Stop(ctx context.Context, unit string) error
|
|
}
|
|
|
|
// Images fetches the agent image a release pins.
|
|
type Images interface {
|
|
Pull(ctx context.Context, ref string) error
|
|
}
|
|
|
|
// ImagePolicy is the control plane's agent image setting.
|
|
type ImagePolicy interface {
|
|
Image(ctx context.Context) (string, error)
|
|
// SetImage replaces expected with desired. It succeeds without writing
|
|
// when desired is already set, and refuses any other current value.
|
|
SetImage(ctx context.Context, expected, desired string) error
|
|
}
|
|
|
|
// Database is the task database the services share.
|
|
type Database interface {
|
|
Backup(ctx context.Context, out string) error
|
|
SchemaVersion(ctx context.Context) (int64, error)
|
|
Restore(ctx context.Context, from string) error
|
|
}
|
|
|
|
// Health asks the daemon whether it is healthy.
|
|
type Health interface {
|
|
// Healthy returns the version a healthy daemon reports, or why it is not.
|
|
Healthy(ctx context.Context) (string, error)
|
|
}
|
|
|
|
// Transaction installs a release and either commits it or rolls the host back
|
|
// to the previous binaries and image policy. Prepare stages everything without
|
|
// touching what runs; Run, in a process of its own, switches over, watches the
|
|
// daemon and decides.
|
|
type Transaction struct {
|
|
StateDir string
|
|
BinDir string
|
|
// ConfigPath is the bootstrap config file the install runs against. Its
|
|
// sibling env file supplies the credentials the systemd units would
|
|
// otherwise provide, above all the database password; empty means the
|
|
// default config path.
|
|
ConfigPath string
|
|
Units Units
|
|
Images Images
|
|
Policy ImagePolicy
|
|
Database Database
|
|
Health Health
|
|
Progress func(string)
|
|
|
|
// HealthTimeout bounds the wait for the restarted daemon to report the
|
|
// release; it must then stay healthy for Observe, checked every Poll.
|
|
HealthTimeout time.Duration
|
|
Observe time.Duration
|
|
Poll time.Duration
|
|
}
|
|
|
|
type imageChange struct {
|
|
Previous string `json:"previous"`
|
|
Candidate string `json:"candidate"`
|
|
}
|
|
|
|
type journal struct {
|
|
Phase string `json:"phase"`
|
|
Version string `json:"version"`
|
|
Previous map[string]string `json:"previous"`
|
|
Units []string `json:"units"`
|
|
Binaries []string `json:"binaries,omitempty"`
|
|
Managed bool `json:"managed"`
|
|
Image imageChange `json:"image"`
|
|
SchemaBefore int64 `json:"schema_before"`
|
|
Meta InstallMeta `json:"meta"`
|
|
// RestoredDatabase records that rollback found a migration and restored
|
|
// the pre-update snapshot.
|
|
RestoredDatabase bool `json:"restored_database"`
|
|
}
|
|
|
|
// JournalPath is where an unfinished transaction lives.
|
|
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
|
|
|
|
// Run drives the transaction in the journal to an outcome. recovering is set
|
|
// on boot: it never waits for the lock, and a transaction that was prepared
|
|
// but never started is abandoned rather than started unattended.
|
|
func (t *Transaction) Run(ctx context.Context, recovering bool) error {
|
|
if err := t.loadEnvCredentials(); err != nil {
|
|
return err
|
|
}
|
|
unlock, held, err := lockState(ctx, t.StateDir, !recovering)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !held {
|
|
// A live installer or watchdog owns the journal.
|
|
return nil
|
|
}
|
|
defer unlock()
|
|
j, found, err := t.load()
|
|
if err != nil || !found {
|
|
return err
|
|
}
|
|
switch j.Phase {
|
|
case phasePrepared:
|
|
if recovering {
|
|
return t.abandon(ctx, j)
|
|
}
|
|
return t.promote(ctx, j)
|
|
case phaseCommitted, phaseRolledBack:
|
|
// The outcome was decided before the watchdog died. archied exits at
|
|
// boot while a journal exists, so the units come up again here.
|
|
if err := t.startUnits(ctx, j, ""); err != nil {
|
|
return err
|
|
}
|
|
return t.remove()
|
|
default:
|
|
return t.rollback(ctx, j)
|
|
}
|
|
}
|
|
|
|
func (t *Transaction) promote(ctx context.Context, j journal) error {
|
|
if err := t.setPhase(&j, phaseSwitching); err != nil {
|
|
return err
|
|
}
|
|
if err := t.switchOver(ctx, &j); err != nil {
|
|
t.say("update to %s failed: %v; rolling back", j.Version, err)
|
|
return t.rollback(ctx, j)
|
|
}
|
|
if j.Managed {
|
|
if err := t.writeReport(j, "passed", false); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
t.say("installed %s; restart the archie services to run it", j.Version)
|
|
}
|
|
if err := t.setPhase(&j, phaseCommitted); err != nil {
|
|
return err
|
|
}
|
|
return t.remove()
|
|
}
|
|
|
|
// switchOver replaces the executables and image policy, then restarts the
|
|
// units in release order and requires the daemon to stay healthy.
|
|
func (t *Transaction) switchOver(ctx context.Context, j *journal) error {
|
|
if err := t.stopUnits(ctx, *j); err != nil {
|
|
return err
|
|
}
|
|
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
|
|
if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil {
|
|
return fmt.Errorf("switch %s: %w", unit, err)
|
|
}
|
|
}
|
|
if err := fsutil.WriteFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n"), 0o600); err != nil {
|
|
return err
|
|
}
|
|
if err := syncDir(t.BinDir); err != nil {
|
|
return err
|
|
}
|
|
if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil {
|
|
return err
|
|
}
|
|
if !j.Managed {
|
|
return nil
|
|
}
|
|
if err := t.setPhase(j, phaseObserving); err != nil {
|
|
return err
|
|
}
|
|
if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil {
|
|
return err
|
|
}
|
|
return t.observe(ctx, j.Version)
|
|
}
|
|
|
|
// rollback is safe to repeat from any point: every step converges on the
|
|
// previous state whether or not an earlier attempt got that far.
|
|
func (t *Transaction) rollback(ctx context.Context, j journal) error {
|
|
if err := t.setPhase(&j, phaseRollingBack); err != nil {
|
|
return err
|
|
}
|
|
if err := t.stopUnits(ctx, j); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
if err := t.restoreFiles(j); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
restored, err := t.restoreDatabase(ctx, j)
|
|
if err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
started := ""
|
|
if restored {
|
|
// The snapshot predates the policy switch, so it holds the previous image.
|
|
j.RestoredDatabase = true
|
|
if err := t.setPhase(&j, phaseRollingBack); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot())
|
|
} else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil {
|
|
return t.incomplete(err)
|
|
} else {
|
|
started = stateStoreUnit
|
|
}
|
|
if err := t.writeReport(j, "failed", true); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
if err := t.setPhase(&j, phaseRolledBack); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
if err := t.startUnits(ctx, j, started); err != nil {
|
|
return t.incomplete(err)
|
|
}
|
|
return t.remove()
|
|
}
|
|
|
|
// abandon discards a transaction that never switched anything.
|
|
func (t *Transaction) abandon(ctx context.Context, j journal) error {
|
|
t.say("update to %s was staged but never started; discarding it", j.Version)
|
|
for _, app := range append(slices.Clone(j.Units), j.Binaries...) {
|
|
if err := os.Remove(t.staged(app)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
}
|
|
if err := t.writeReport(j, "failed", true); err != nil {
|
|
return err
|
|
}
|
|
if err := t.setPhase(&j, phaseRolledBack); err != nil {
|
|
return err
|
|
}
|
|
if err := t.startUnits(ctx, j, ""); err != nil {
|
|
return err
|
|
}
|
|
return t.remove()
|
|
}
|
|
|
|
func (t *Transaction) incomplete(err error) error {
|
|
return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err)
|
|
}
|
|
|
|
func (t *Transaction) restoreFiles(j journal) error {
|
|
previous := filepath.Join(JournalPath(t.StateDir), "previous")
|
|
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
|
|
live := filepath.Join(t.BinDir, unit)
|
|
saved := filepath.Join(previous, unit)
|
|
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
|
|
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
} else if err := installFile(saved, live, 0o755); err != nil {
|
|
return fmt.Errorf("restore %s: %w", unit, err)
|
|
}
|
|
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
}
|
|
version := filepath.Join(t.BinDir, agentVersionFile)
|
|
saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile)
|
|
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
|
|
if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
return err
|
|
}
|
|
} else if err := installFile(saved, version, 0o644); err != nil {
|
|
return err
|
|
}
|
|
return syncDir(t.BinDir)
|
|
}
|
|
|
|
// restoreDatabase restores the pre-update snapshot only when the candidate
|
|
// migrated the schema: the previous binaries refuse a newer schema, and data
|
|
// written since the snapshot is otherwise worth keeping.
|
|
func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) {
|
|
if j.RestoredDatabase {
|
|
return true, nil
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
|
schema, err := t.Database.SchemaVersion(callCtx)
|
|
cancel()
|
|
if err != nil {
|
|
return false, fmt.Errorf("read schema version: %w", err)
|
|
}
|
|
if schema <= j.SchemaBefore {
|
|
return false, nil
|
|
}
|
|
if err := t.Database.Restore(ctx, t.snapshot()); err != nil {
|
|
return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// setImage retries while the State Store comes up. No attempt outlives the
|
|
// deadline, so a peer that never answers still ends in rollback.
|
|
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
|
|
if j.Managed {
|
|
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
deadline := time.Now().Add(t.HealthTimeout)
|
|
for {
|
|
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
|
err := t.Policy.SetImage(callCtx, expected, desired)
|
|
cancel()
|
|
if err == nil || time.Now().After(deadline) {
|
|
return err
|
|
}
|
|
if err := sleep(ctx, t.Poll); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
func (t *Transaction) observe(ctx context.Context, version string) error {
|
|
deadline := time.Now().Add(t.HealthTimeout)
|
|
for {
|
|
callCtx, cancel := context.WithDeadline(ctx, deadline)
|
|
reported, err := t.Health.Healthy(callCtx)
|
|
cancel()
|
|
if err == nil && sameVersion(reported, version) {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err))
|
|
}
|
|
if err := sleep(ctx, t.Poll); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
t.say("archied is healthy on %s; watching it for %s", version, t.Observe)
|
|
until := time.Now().Add(t.Observe)
|
|
for time.Now().Before(until) {
|
|
if err := sleep(ctx, t.Poll); err != nil {
|
|
return err
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
|
|
reported, err := t.Health.Healthy(callCtx)
|
|
cancel()
|
|
if err != nil || !sameVersion(reported, version) {
|
|
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func healthProblem(reported string, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return fmt.Errorf("it reports version %q", reported)
|
|
}
|
|
|
|
func sameVersion(left, right string) bool {
|
|
l, ok := comparableVersion(left)
|
|
r, _ := comparableVersion(right)
|
|
return ok && l == r
|
|
}
|
|
|
|
// startUnits restarts the units in release order, skipping one already started.
|
|
func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error {
|
|
if !j.Managed {
|
|
return nil
|
|
}
|
|
for _, unit := range j.Units {
|
|
if unit == skip {
|
|
continue
|
|
}
|
|
if err := t.Units.Start(ctx, unit); err != nil {
|
|
return fmt.Errorf("start %s: %w", unit, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// stopUnits stops in reverse release order, so callers go before the services
|
|
// they depend on.
|
|
func (t *Transaction) stopUnits(ctx context.Context, j journal) error {
|
|
if !j.Managed {
|
|
return nil
|
|
}
|
|
for _, unit := range slices.Backward(j.Units) {
|
|
if err := t.Units.Stop(ctx, unit); err != nil {
|
|
return fmt.Errorf("stop %s: %w", unit, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error {
|
|
if j.Meta.ReportPath == "" {
|
|
return nil
|
|
}
|
|
installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version}
|
|
return WritePendingReport(j.Meta.ReportPath, Report{
|
|
Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID,
|
|
Previous: j.Previous, Installed: installed,
|
|
HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase,
|
|
})
|
|
}
|
|
|
|
func (t *Transaction) staged(unit string) string {
|
|
return filepath.Join(t.BinDir, "."+unit+".new")
|
|
}
|
|
|
|
func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) }
|
|
|
|
func (t *Transaction) say(format string, args ...any) {
|
|
if t.Progress != nil {
|
|
t.Progress(fmt.Sprintf(format, args...))
|
|
}
|
|
}
|
|
|
|
func (t *Transaction) load() (journal, bool, error) {
|
|
data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile))
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
return journal{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return journal{}, false, fmt.Errorf("read update journal: %w", err)
|
|
}
|
|
var j journal
|
|
if err := json.Unmarshal(data, &j); err != nil {
|
|
return journal{}, false, fmt.Errorf("decode update journal: %w", err)
|
|
}
|
|
return j, true, nil
|
|
}
|
|
|
|
func (t *Transaction) setPhase(j *journal, phase string) error {
|
|
j.Phase = phase
|
|
return writeJournal(JournalPath(t.StateDir), *j)
|
|
}
|
|
|
|
func writeJournal(dir string, j journal) error {
|
|
data, err := json.Marshal(j)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := fsutil.WriteFileAtomic(filepath.Join(dir, journalFile), data, 0o600); err != nil {
|
|
return err
|
|
}
|
|
return syncDir(dir)
|
|
}
|
|
|
|
// remove retires the journal by rename first, so a crash mid-delete never
|
|
// leaves a partial journal behind.
|
|
func (t *Transaction) remove() error {
|
|
finished := filepath.Join(t.StateDir, "finished")
|
|
if err := os.RemoveAll(finished); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(JournalPath(t.StateDir), finished); err != nil {
|
|
return err
|
|
}
|
|
if err := syncDir(t.StateDir); err != nil {
|
|
return err
|
|
}
|
|
return os.RemoveAll(finished)
|
|
}
|
|
|
|
// lockState takes the update lock. Without wait it reports held=false at once
|
|
// when another process owns it.
|
|
func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, false, err
|
|
}
|
|
file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
deadline := time.Now().Add(lockWait)
|
|
for {
|
|
err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
|
|
if err == nil {
|
|
return func() { _ = file.Close() }, true, nil
|
|
}
|
|
if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) {
|
|
_ = file.Close()
|
|
if errors.Is(err, unix.EWOULDBLOCK) {
|
|
return nil, false, nil
|
|
}
|
|
return nil, false, err
|
|
}
|
|
if err := sleep(ctx, 100*time.Millisecond); err != nil {
|
|
_ = file.Close()
|
|
return nil, false, err
|
|
}
|
|
}
|
|
}
|
|
|
|
func sleep(ctx context.Context, d time.Duration) error {
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func syncDir(dir string) error {
|
|
file, err := os.Open(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = file.Close() }()
|
|
return file.Sync()
|
|
}
|
|
|
|
// installFile copies src over dst through a synced temp file and rename, so
|
|
// dst is always either the old or the new content.
|
|
func installFile(src, dst string, mode os.FileMode) error {
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp")
|
|
if err := writeSynced(tmp, data, mode); err != nil {
|
|
return err
|
|
}
|
|
return os.Rename(tmp, dst)
|
|
}
|
|
|
|
func writeSynced(path string, data []byte, mode os.FileMode) error {
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := file.Write(data); err != nil {
|
|
_ = file.Close()
|
|
return err
|
|
}
|
|
if err := file.Chmod(mode); err != nil {
|
|
_ = file.Close()
|
|
return err
|
|
}
|
|
if err := file.Sync(); err != nil {
|
|
_ = file.Close()
|
|
return err
|
|
}
|
|
return file.Close()
|
|
}
|