Files
sam 073959b4f9 feat(memory): scope chat notes and preserve deletions on restore (#2069)
Keep personal notes private by default and require review before publishing shared notes. Restore backups and replay forgotten notes in one transaction, and retain the installed release when an unattended rollback could bring back forgotten notes.
2026-11-21 13:20:31 +11:00

603 lines
18 KiB
Go

package releaseupdate
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"slices"
"time"
"golang.org/x/sys/unix"
"github.com/samcharles93/archie-core/internal/fsutil"
"github.com/samcharles93/archie-core/internal/platform/deletions"
)
const (
phasePrepared = "prepared"
phaseSwitching = "switching"
phaseObserving = "observing"
phaseRollingBack = "rolling-back"
phaseCommitted = "committed"
phaseRolledBack = "rolled-back"
phaseRollbackRefused = "rollback-refused"
)
const (
// stateStoreUnit is started ahead of the others so the image policy can
// be switched before archied reads it.
stateStoreUnit = "archie-state-store"
agentVersionFile = "archie-agent.version"
// WatchdogBinary is the archied copy frozen into the journal: the binary
// that wrote a journal is the one that interprets it, even after the
// executables in the bin directory have been replaced.
WatchdogBinary = "archied"
journalDir = "transaction"
journalFile = "journal.json"
snapshotFile = "db.pre-update.dump"
lockWait = 60 * time.Second
)
// Units starts and stops the host's services. Managed is false where no
// service manager runs them; then nothing is restarted.
type Units interface {
Managed(ctx context.Context) bool
Start(ctx context.Context, unit string) error
Stop(ctx context.Context, unit string) error
}
// Images fetches the agent image a release pins.
type Images interface {
Pull(ctx context.Context, ref string) error
}
// ImagePolicy is the control plane's agent image setting.
type ImagePolicy interface {
Image(ctx context.Context) (string, error)
// SetImage replaces expected with desired. It succeeds without writing
// when desired is already set, and refuses any other current value.
SetImage(ctx context.Context, expected, desired string) error
}
// Database is the task database the services share.
type Database interface {
Backup(ctx context.Context, out string) error
SchemaVersion(ctx context.Context) (int64, error)
Restore(ctx context.Context, from string) error
}
// Health asks the daemon whether it is healthy.
type Health interface {
// Healthy returns the version a healthy daemon reports, or why it is not.
Healthy(ctx context.Context) (string, error)
}
// Transaction installs a release and either commits it or rolls the host back
// to the previous binaries and image policy. Prepare stages everything without
// touching what runs; Run, in a process of its own, switches over, watches the
// daemon and decides.
type Transaction struct {
StateDir string
BinDir string
// ConfigPath is the bootstrap config file the install runs against. Its
// sibling env file supplies the credentials the systemd units would
// otherwise provide, above all the database password; empty means the
// default config path.
ConfigPath string
Units Units
Images Images
Policy ImagePolicy
Database Database
Health Health
Progress func(string)
// HealthTimeout bounds the wait for the restarted daemon to report the
// release; it must then stay healthy for Observe, checked every Poll.
HealthTimeout time.Duration
Observe time.Duration
Poll time.Duration
}
type imageChange struct {
Previous string `json:"previous"`
Candidate string `json:"candidate"`
}
type journal struct {
Phase string `json:"phase"`
Version string `json:"version"`
Previous map[string]string `json:"previous"`
Units []string `json:"units"`
Binaries []string `json:"binaries,omitempty"`
Managed bool `json:"managed"`
Image imageChange `json:"image"`
SchemaBefore int64 `json:"schema_before"`
Meta InstallMeta `json:"meta"`
// RestoredDatabase records that rollback found a migration and restored
// the pre-update snapshot.
RestoredDatabase bool `json:"restored_database"`
}
// JournalPath is where an unfinished transaction lives.
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
// Run drives the transaction in the journal to an outcome. recovering is set
// on boot: it never waits for the lock, and a transaction that was prepared
// but never started is abandoned rather than started unattended.
func (t *Transaction) Run(ctx context.Context, recovering bool) error {
if err := t.loadEnvCredentials(); err != nil {
return err
}
unlock, held, err := lockState(ctx, t.StateDir, !recovering)
if err != nil {
return err
}
if !held {
// A live installer or watchdog owns the journal.
return nil
}
defer unlock()
j, found, err := t.load()
if err != nil || !found {
return err
}
switch j.Phase {
case phasePrepared:
if recovering {
return t.abandon(ctx, j)
}
return t.promote(ctx, j)
case phaseCommitted, phaseRolledBack, phaseRollbackRefused:
// The outcome was decided before the watchdog died. archied exits at
// boot while a journal exists, so the units come up again here.
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
default:
return t.rollback(ctx, j)
}
}
func (t *Transaction) promote(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseSwitching); err != nil {
return err
}
if err := t.switchOver(ctx, &j); err != nil {
t.say("update to %s failed: %v; rolling back", j.Version, err)
return t.rollback(ctx, j)
}
if j.Managed {
if err := t.writeReport(j, "passed", false); err != nil {
return err
}
} else {
t.say("installed %s; restart the archie services to run it", j.Version)
}
if err := t.setPhase(&j, phaseCommitted); err != nil {
return err
}
return t.remove()
}
// switchOver replaces the executables and image policy, then restarts the
// units in release order and requires the daemon to stay healthy.
func (t *Transaction) switchOver(ctx context.Context, j *journal) error {
if err := t.stopUnits(ctx, *j); err != nil {
return err
}
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil {
return fmt.Errorf("switch %s: %w", unit, err)
}
}
if err := fsutil.WriteFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n"), 0o600); err != nil {
return err
}
if err := syncDir(t.BinDir); err != nil {
return err
}
if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil {
return err
}
if !j.Managed {
return nil
}
if err := t.setPhase(j, phaseObserving); err != nil {
return err
}
if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil {
return err
}
return t.observe(ctx, j.Version)
}
func (t *Transaction) rollback(ctx context.Context, j journal) error {
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return err
}
if err := t.stopUnits(ctx, j); err != nil {
return t.incomplete(err)
}
// If the database can't be restored safely, the previous binaries may not
// support its schema. Leave the candidate installed until the check succeeds.
restored, err := t.restoreDatabase(ctx, j)
if errors.Is(err, deletions.ErrUnsafeRollback) {
return t.retainCandidate(ctx, j, err)
}
if err != nil {
return t.incomplete(err)
}
if restored && !j.RestoredDatabase {
j.RestoredDatabase = true
if err := t.setPhase(&j, phaseRollingBack); err != nil {
return t.incomplete(err)
}
t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot())
}
if err := t.restoreFiles(j); err != nil {
return t.incomplete(err)
}
started := ""
// The snapshot predates the policy switch, so it holds the previous image.
if !restored {
if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil {
return t.incomplete(err)
}
started = stateStoreUnit
}
if err := t.writeReport(j, "failed", true); err != nil {
return t.incomplete(err)
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return t.incomplete(err)
}
if err := t.startUnits(ctx, j, started); err != nil {
return t.incomplete(err)
}
return t.remove()
}
func (t *Transaction) retainCandidate(ctx context.Context, j journal, reason error) error {
t.say("keeping release %s installed: %v; manual recovery is required", j.Version, reason)
if err := t.writeReport(j, "failed", false); err != nil {
return t.incomplete(err)
}
if err := t.setPhase(&j, phaseRollbackRefused); err != nil {
return t.incomplete(err)
}
if err := t.startUnits(ctx, j, ""); err != nil {
return t.incomplete(err)
}
return t.remove()
}
// abandon discards a transaction that never switched anything.
func (t *Transaction) abandon(ctx context.Context, j journal) error {
t.say("update to %s was staged but never started; discarding it", j.Version)
for _, app := range append(slices.Clone(j.Units), j.Binaries...) {
if err := os.Remove(t.staged(app)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
if err := t.writeReport(j, "failed", true); err != nil {
return err
}
if err := t.setPhase(&j, phaseRolledBack); err != nil {
return err
}
if err := t.startUnits(ctx, j, ""); err != nil {
return err
}
return t.remove()
}
func (t *Transaction) incomplete(err error) error {
return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err)
}
func (t *Transaction) restoreFiles(j journal) error {
previous := filepath.Join(JournalPath(t.StateDir), "previous")
for _, unit := range append(slices.Clone(j.Units), j.Binaries...) {
live := filepath.Join(t.BinDir, unit)
saved := filepath.Join(previous, unit)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, live, 0o755); err != nil {
return fmt.Errorf("restore %s: %w", unit, err)
}
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
}
version := filepath.Join(t.BinDir, agentVersionFile)
saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile)
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
} else if err := installFile(saved, version, 0o644); err != nil {
return err
}
return syncDir(t.BinDir)
}
// restoreDatabase restores the pre-update snapshot only when the candidate
// migrated the schema: the previous binaries refuse a newer schema, and data
// written since the snapshot is otherwise worth keeping.
func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) {
if j.RestoredDatabase {
return true, nil
}
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
schema, err := t.Database.SchemaVersion(callCtx)
cancel()
if err != nil {
return false, fmt.Errorf("read schema version: %w", err)
}
if schema <= j.SchemaBefore {
return false, nil
}
if err := t.Database.Restore(ctx, t.snapshot()); err != nil {
return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err)
}
return true, nil
}
// setImage retries while the State Store comes up. No attempt outlives the
// deadline, so a peer that never answers still ends in rollback.
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
if j.Managed {
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
return err
}
}
deadline := time.Now().Add(t.HealthTimeout)
for {
callCtx, cancel := context.WithDeadline(ctx, deadline)
err := t.Policy.SetImage(callCtx, expected, desired)
cancel()
if err == nil || time.Now().After(deadline) {
return err
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
}
func (t *Transaction) observe(ctx context.Context, version string) error {
deadline := time.Now().Add(t.HealthTimeout)
for {
callCtx, cancel := context.WithDeadline(ctx, deadline)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err == nil && sameVersion(reported, version) {
break
}
if time.Now().After(deadline) {
return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err))
}
if err := sleep(ctx, t.Poll); err != nil {
return err
}
}
t.say("archied is healthy on %s; watching it for %s", version, t.Observe)
until := time.Now().Add(t.Observe)
for time.Now().Before(until) {
if err := sleep(ctx, t.Poll); err != nil {
return err
}
callCtx, cancel := context.WithTimeout(ctx, t.HealthTimeout)
reported, err := t.Health.Healthy(callCtx)
cancel()
if err != nil || !sameVersion(reported, version) {
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
}
}
return nil
}
func healthProblem(reported string, err error) error {
if err != nil {
return err
}
return fmt.Errorf("it reports version %q", reported)
}
func sameVersion(left, right string) bool {
l, ok := comparableVersion(left)
r, _ := comparableVersion(right)
return ok && l == r
}
// startUnits restarts the units in release order, skipping one already started.
func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error {
if !j.Managed {
return nil
}
for _, unit := range j.Units {
if unit == skip {
continue
}
if err := t.Units.Start(ctx, unit); err != nil {
return fmt.Errorf("start %s: %w", unit, err)
}
}
return nil
}
// stopUnits stops in reverse release order, so callers go before the services
// they depend on.
func (t *Transaction) stopUnits(ctx context.Context, j journal) error {
if !j.Managed {
return nil
}
for _, unit := range slices.Backward(j.Units) {
if err := t.Units.Stop(ctx, unit); err != nil {
return fmt.Errorf("stop %s: %w", unit, err)
}
}
return nil
}
func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error {
if j.Meta.ReportPath == "" {
return nil
}
installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version}
return WritePendingReport(j.Meta.ReportPath, Report{
Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID,
Previous: j.Previous, Installed: installed,
HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase,
})
}
func (t *Transaction) staged(unit string) string {
return filepath.Join(t.BinDir, "."+unit+".new")
}
func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) }
func (t *Transaction) say(format string, args ...any) {
if t.Progress != nil {
t.Progress(fmt.Sprintf(format, args...))
}
}
func (t *Transaction) load() (journal, bool, error) {
data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile))
if errors.Is(err, os.ErrNotExist) {
return journal{}, false, nil
}
if err != nil {
return journal{}, false, fmt.Errorf("read update journal: %w", err)
}
var j journal
if err := json.Unmarshal(data, &j); err != nil {
return journal{}, false, fmt.Errorf("decode update journal: %w", err)
}
return j, true, nil
}
func (t *Transaction) setPhase(j *journal, phase string) error {
j.Phase = phase
return writeJournal(JournalPath(t.StateDir), *j)
}
func writeJournal(dir string, j journal) error {
data, err := json.Marshal(j)
if err != nil {
return err
}
if err := fsutil.WriteFileAtomic(filepath.Join(dir, journalFile), data, 0o600); err != nil {
return err
}
return syncDir(dir)
}
// remove retires the journal by rename first, so a crash mid-delete never
// leaves a partial journal behind.
func (t *Transaction) remove() error {
finished := filepath.Join(t.StateDir, "finished")
if err := os.RemoveAll(finished); err != nil {
return err
}
if err := os.Rename(JournalPath(t.StateDir), finished); err != nil {
return err
}
if err := syncDir(t.StateDir); err != nil {
return err
}
return os.RemoveAll(finished)
}
// lockState takes the update lock. Without wait it reports held=false at once
// when another process owns it.
func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) {
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, false, err
}
file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, false, err
}
deadline := time.Now().Add(lockWait)
for {
err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err == nil {
return func() { _ = file.Close() }, true, nil
}
if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) {
_ = file.Close()
if errors.Is(err, unix.EWOULDBLOCK) {
return nil, false, nil
}
return nil, false, err
}
if err := sleep(ctx, 100*time.Millisecond); err != nil {
_ = file.Close()
return nil, false, err
}
}
}
func sleep(ctx context.Context, d time.Duration) error {
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func syncDir(dir string) error {
file, err := os.Open(dir)
if err != nil {
return err
}
defer func() { _ = file.Close() }()
return file.Sync()
}
// installFile copies src over dst through a synced temp file and rename, so
// dst is always either the old or the new content.
func installFile(src, dst string, mode os.FileMode) error {
data, err := os.ReadFile(src)
if err != nil {
return err
}
tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp")
if err := writeSynced(tmp, data, mode); err != nil {
return err
}
return os.Rename(tmp, dst)
}
func writeSynced(path string, data []byte, mode os.FileMode) error {
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
if err != nil {
return err
}
if _, err := file.Write(data); err != nil {
_ = file.Close()
return err
}
if err := file.Chmod(mode); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
return file.Close()
}