diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 223dfba8a..4c958a708 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -372,7 +372,6 @@ jobs: cp "dist/$cmd" "$name/" done cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/" - cp scripts/archie-update-* "$name/" digest='${{ needs.agent-image.outputs.agent_digest }}' test -n "$digest" jq --arg image "ghcr.io/samcharles93/archie-agent@$digest" '. + {agent_image: $image}' release.json > "$name/release.json" diff --git a/Taskfile.yml b/Taskfile.yml index 294f7f199..3be922e81 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -285,8 +285,8 @@ tasks: - "{{.BIN_DIR}}/archie-messaging" - "{{.BIN_DIR}}/archie-agent" cmds: - # One flag set for every service, matching scripts/archie-update-install - # and the release workflow, so each binary reports the same version. + # One flag set for every service, matching the release workflow, so each + # binary reports the same version. - for: [archied, archie-gateway, archie-state-store, archie-ui, archie-messaging] cmd: go build -ldflags "{{.SERVICE_LDFLAGS}}" -o {{.BIN_DIR}}/{{.ITEM}} ./cmd/{{.ITEM}} - go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/agentworker.version={{.RUNTIME_VERSION}} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" -o {{.BIN_DIR}}/archie-agent ./cmd/archie-agent diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 8c0c945ef..6c6e190d3 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -20,12 +20,6 @@ func main() { if len(args) > 0 && args[0] == "update" { os.Exit(archied.RunUpdate(args[1:], os.Stdout, os.Stderr)) } - if len(args) > 0 && args[0] == "update-image" { - os.Exit(archied.RunUpdateImage(args[1:], os.Stderr)) - } - if len(args) > 0 && args[0] == "update-topology" { - os.Exit(archied.RunUpdateTopology(args[1:], os.Stdout, os.Stderr)) - } if len(args) > 0 && args[0] == "status" { os.Exit(archied.RunStatus(args[1:], os.Stdout, os.Stderr)) } diff --git a/deployments/INSTRUCTIONS.md b/deployments/INSTRUCTIONS.md index 8b49494e3..3f446f647 100644 --- a/deployments/INSTRUCTIONS.md +++ b/deployments/INSTRUCTIONS.md @@ -90,12 +90,14 @@ an install (or an upgrade across v1.30.0) needs its unit as well -- see ### Native self-update -`archie-update-install` downloads the approved release's Linux/amd64 archive -and verifies its `SHA256SUMS` entry before executing or installing it. Releases -must include `release.json`, all host binaries, and the two updater scripts. -Older releases without this contract cannot be installed through the artifact -path. `--from-source` (or `ARCHIE_UPDATE_SOURCE_BUILD=1`) explicitly clones and -builds the selected release instead. +Install on Settings → Status runs `archied update install -version X`, which +downloads the release's Linux/amd64 zip and verifies it against `SHA256SUMS` +before unpacking it. Every host binary in the zip must report `X`. It then pulls +the agent image `release.json` pins by digest, backs up the database to +`db.pre-update.dump` in the state directory, stages the new binaries beside the +live ones and hands the transaction to `archied update watchdog` in a process of +its own (a transient systemd user unit). Nothing that runs changes until the +watchdog takes over. `archied update check` finds the newest GitHub release with a linux-amd64 zip on the channel chosen under Settings → Status → Updates: stable, next (includes @@ -103,30 +105,32 @@ prereleases) or a pinned version. From a shell it reads `ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`, or `-channel` and `-pin`. The notification destination remains separate from the release channel. -For scheduled, unattended runs, `archie-update-install --auto` discovers the -selected release and refuses if the daemon reports work in flight. It checks -again immediately before promotion. A scheduler can set -`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`; nothing schedules updates -by default. All services stop through their ordinary graceful shutdown path. - `release.json` supplies the service order and required config sections to both first-install and self-update. A missing unit or config section produces an ordered migration plan and refuses installation. Apply that plan before retrying. Automatic topology edits are intentionally unsupported. -The watchdog switches staged executables by rename, restarts services, and -requires the candidate daemon's existing readiness registry to remain healthy -for 120 seconds. Agent images come from the release manifest by digest. The -image switch updates the control-plane policy, preserving other settings; -rollback restores the prior image policy with conflict detection. Source builds -use the built image's immutable image ID. +The watchdog stops the units in reverse `release.json` order, switches the +staged executables by rename, switches the control-plane image policy (only the +image field, with conflict detection), and restarts the units in order through +`systemctl --user`. archied must report the new version and stay healthy for +120 seconds. Otherwise the watchdog restores the previous binaries and image +policy. Data is not rolled back unless the new release migrated the database: +then the previous binaries cannot serve it, so the pre-update backup is restored +and the outcome report says that every write since the update began is gone. +Without a systemd user manager the watchdog switches binaries and image policy +but restarts nothing, and the install reports that the services need a manual +restart. Transactions live in `${XDG_STATE_HOME:-~/.local/state}/archie/update`, outside -the binary directory. `phase`, `environment`, `image.json` (when applicable), -and `previous/` describe the transaction. A lock reports the owning process. -On daemon boot, an interrupted transaction schedules independent recovery before -normal startup. To recover manually, run `transaction/watchdog --recover` from -that state directory. A failed recovery keeps the journal for another attempt. +the binary directory: `transaction/journal.json`, `transaction/previous/`, and +`transaction/archied`, the frozen copy of the archied that wrote the journal. On +daemon boot, an interrupted transaction is +handed to that frozen watchdog with `-recover` before normal startup: a +transaction that never started is discarded, one that was switching or observing +is rolled back, and a decided one is finished. To recover manually, run +`transaction/archied update watchdog -recover -bin-dir ~/.local/bin` from that +state directory. A failed recovery keeps the journal for another attempt. -Database snapshots remain available for manual recovery. Schema migrations and -PostgreSQL data are never automatically reversed. +The database backup stays in the state directory after a committed update for +manual recovery with `archie-state-store restore`. diff --git a/deployments/systemd-user-service.md b/deployments/systemd-user-service.md index 200be5597..0dcfc126e 100644 --- a/deployments/systemd-user-service.md +++ b/deployments/systemd-user-service.md @@ -36,7 +36,7 @@ by `archied`, since the v1.30.0 extraction. `archied` starts no gateway for them and logs nothing when one is absent, so a host that updates across v1.30.0 without this unit runs on with dead channels and a daemon that looks perfectly healthy. `install.sh` writes this unit on a fresh install; -`scripts/archie-update-install` refuses an +`archied update install` refuses an update when a host is missing a unit and names what is missing, which now fires for a host that drifted or predates this release rather than the normal path. diff --git a/install.sh b/install.sh index f34d133d2..8b9a301e0 100755 --- a/install.sh +++ b/install.sh @@ -89,8 +89,8 @@ fi mkdir -p "${ARCHIE_CONFIG_DIR}" "${ARCHIE_DATA_DIR}" "${ARCHIE_BIN_DIR}" -# 2. Binaries. STAGE ends up holding the binaries, release.json and the -# updater scripts, whichever way they were obtained. +# 2. Binaries. STAGE ends up holding the binaries and release.json, whichever +# way they were obtained. TMP="$(mktemp -d)" trap 'rm -rf "${TMP}"' EXIT @@ -111,7 +111,7 @@ if [ "${FROM_SOURCE}" = true ]; then for cmd in $(jq -er '.required_topology.units | join(" ")' "${SRC_DIR}/release.json"); do (cd "${SRC_DIR}" && go build -ldflags "${LDFLAGS}" -o "${STAGE}/${cmd}" "./cmd/${cmd}") done - cp "${SRC_DIR}/release.json" "${SRC_DIR}"/scripts/archie-update-* "${STAGE}/" + cp "${SRC_DIR}/release.json" "${STAGE}/" else if [ -z "${VERSION}" ]; then tag="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" | sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p')" @@ -137,8 +137,7 @@ TOPOLOGY_UNITS="$(tr -d '\n' <"${STAGE}/release.json" | sed -n 's/.*"units": *\[ for cmd in ${TOPOLOGY_UNITS}; do install -m755 "${STAGE}/${cmd}" "${ARCHIE_BIN_DIR}/${cmd}" done -install -m755 "${STAGE}"/archie-update-* "${ARCHIE_BIN_DIR}/" -echo " Installed ${TOPOLOGY_UNITS} and the updater to ${ARCHIE_BIN_DIR}/" +echo " Installed ${TOPOLOGY_UNITS} to ${ARCHIE_BIN_DIR}/" # 3. Configuration. archied setup asks every question and writes config.toml # plus any secrets to the env file beside it; this script asks nothing itself. diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index cb12151ec..102ab7481 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -126,7 +126,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence return 0 } - if recovering, err := resumeUpdate(context.Background()); err != nil { + if recovering, err := resumeUpdate(context.Background(), args.cfgPath); err != nil { fmt.Fprintln(os.Stderr, err) return 1 } else if recovering { diff --git a/internal/app/archied/update.go b/internal/app/archied/update.go index 2f5e2b201..7fd530b85 100644 --- a/internal/app/archied/update.go +++ b/internal/app/archied/update.go @@ -7,55 +7,200 @@ import ( "flag" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" - "regexp" "strings" + "syscall" "time" "github.com/BurntSushi/toml" "golang.org/x/sys/unix" + "github.com/samcharles93/archie-core/internal/app/servicekit" + "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/releaseupdate" ) -// RunUpdateTopology observes the host using the candidate's topology contract. -func RunUpdateTopology(args []string, out, stderr io.Writer) int { - flags := flag.NewFlagSet("update-topology", flag.ContinueOnError) +// The watchdog's patience: how long the restarted daemon has to report the +// release, and how long it must then stay healthy. +const ( + updateHealthTimeout = 60 * time.Second + updateObserve = 120 * time.Second + updatePoll = 3 * time.Second +) + +const updateUsage = `usage: + archied update check [-channel stable|next|exact-pin] [-pin version] + archied update install -version X [-config FILE] [-report FILE -channel C -chat ID -thread ID] + archied update watchdog [-config FILE] [-recover]` + +// RunUpdate serves `archied update`: check prints the release snapshot, +// install verifies and stages a release and hands it to the watchdog, and +// watchdog switches over, watches the daemon and commits or rolls back. +func RunUpdate(args []string, out, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, updateUsage) + return 2 + } + switch args[0] { + case "check": + return runUpdateCheck(args[1:], out, stderr) + case "install": + return runUpdateInstall(args[1:], out, stderr) + case "watchdog": + return runUpdateWatchdog(args[1:], stderr) + default: + fmt.Fprintln(stderr, updateUsage) + return 2 + } +} + +func runUpdateInstall(args []string, out, stderr io.Writer) int { + flags := flag.NewFlagSet("update install", flag.ContinueOnError) flags.SetOutput(stderr) - configPath := flags.String("config", "", "bootstrap config file") - releasePath := flags.String("release", "", "release manifest") + configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") + version := flags.String("version", "", "release version to install") + releaseURL := flags.String("release-url", releaseupdate.ReleaseURL, "base URL of the release downloads") + var meta releaseupdate.InstallMeta + flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome") + flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on") + flags.Int64Var(&meta.ChatID, "chat", 0, "chat the outcome is reported to") + flags.IntVar(&meta.ThreadID, "thread", 0, "thread the outcome is reported to") if err := flags.Parse(args); err != nil { + return 2 + } + self, err := os.Executable() + if err != nil { + fmt.Fprintln(stderr, err) return 1 } - if err := checkUpdateTopology(*configPath, *releasePath, out); err != nil { + ctx := context.Background() + tx, err := hostTransaction(*configPath, filepath.Dir(self), func(line string) { fmt.Fprintln(out, line) }) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{ + Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta, + CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error { + return checkUpdateTopology(ctx, *configPath, required, managed, out) + }, + }) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary) + if err := spawnWatchdog(ctx, result.RestartRequested, frozen, *configPath, false); err != nil { + // The journal stays: the next archied boot recovers it. + fmt.Fprintln(stderr, err) + return 1 + } + if !result.RestartRequested { + fmt.Fprintf(out, "no systemd user manager: the watchdog installs %s without restarting; restart the archie services to run it\n", *version) + } + data, err := json.Marshal(result) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintln(out, releaseupdate.ResultSentinel+string(data)) + return 0 +} + +func runUpdateWatchdog(args []string, stderr io.Writer) int { + flags := flag.NewFlagSet("update watchdog", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") + recovering := flags.Bool("recover", false, "recover an interrupted transaction at boot") + binDir := flags.String("bin-dir", "", "directory of the live binaries") + if err := flags.Parse(args); err != nil { + return 2 + } + tx, err := hostTransaction(*configPath, *binDir, func(line string) { fmt.Fprintln(stderr, line) }) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + if err := tx.Run(context.Background(), *recovering); err != nil { fmt.Fprintln(stderr, err) return 1 } return 0 } -func checkUpdateTopology(configPath, releasePath string, out io.Writer) error { - data, err := os.ReadFile(releasePath) +// hostTransaction wires the transaction to this host: systemd user units, +// Docker, the daemon's health endpoint, the control-plane image policy and +// the State Store's own backup and restore. +func hostTransaction(configPath, binDir string, progress func(string)) (*releaseupdate.Transaction, error) { + _, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "") + if err != nil { + return nil, err + } + if binDir == "" { + return nil, errors.New("-bin-dir is required") + } + return &releaseupdate.Transaction{ + StateDir: updateStateDir(), + BinDir: binDir, + Units: systemdUnits{}, + Images: dockerImages{}, + Policy: imagePolicy{configPath: configPath}, + Database: stateDatabase{configPath: configPath}, + Health: daemonHealth{url: doc.Config.Health.URL()}, + Progress: progress, + HealthTimeout: updateHealthTimeout, + Observe: updateObserve, + Poll: updatePoll, + }, nil +} + +// spawnWatchdog runs the frozen watchdog outside this process, so restarting +// any service, including the one that asked for the update, cannot kill it. +// Under systemd it is a transient unit of its own; without one it is a +// detached session. +func spawnWatchdog(ctx context.Context, managed bool, binary, configPath string, recovering bool) error { + exe, err := os.Executable() if err != nil { return err } - var manifest struct { - Required releaseupdate.Topology `json:"required_topology"` + args := []string{"update", "watchdog", "-config", configPath, "-bin-dir", filepath.Dir(exe)} + if recovering { + args = append(args, "-recover") } - if err := json.Unmarshal(data, &manifest); err != nil { + stateEnv := "ARCHIE_UPDATE_STATE_DIR=" + updateStateDir() + if managed { + // The units load the env file beside the config for database and + // service credentials; the watchdog needs the same ones. + envFile := "EnvironmentFile=-" + filepath.Join(filepath.Dir(configPath), "env") + run := append([]string{"--user", "--collect", "--quiet", "--unit=archied-update-watchdog", "--setenv=" + stateEnv, "--property=" + envFile, binary}, args...) + if output, err := exec.CommandContext(ctx, "systemd-run", run...).CombinedOutput(); err != nil { + return fmt.Errorf("start the update watchdog: %w: %s", err, output) + } + return nil + } + logFile, err := os.OpenFile(filepath.Join(updateStateDir(), "watchdog.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { return err } - if len(manifest.Required.Units) == 0 { - return errors.New("release has no required_topology") + defer func() { _ = logFile.Close() }() + cmd := exec.Command(binary, args...) //nolint:noctx // the watchdog must outlive this process + cmd.Env = append(os.Environ(), stateEnv) + cmd.Stdout, cmd.Stderr = logFile, logFile + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + if err := cmd.Start(); err != nil { + return fmt.Errorf("start the update watchdog: %w", err) } - for _, unit := range manifest.Required.Units { - if !regexp.MustCompile(`^archie[a-z0-9-]*$`).MatchString(unit) { - return fmt.Errorf("invalid release unit %q", unit) - } - } - data, err = os.ReadFile(configPath) + return cmd.Process.Release() +} + +// checkUpdateTopology refuses a host missing a unit or config section the +// release requires. Units are observable only under systemd; without it there +// is nothing to restart, so only the config is checked. +func checkUpdateTopology(ctx context.Context, configPath string, required releaseupdate.Topology, managed bool, out io.Writer) error { + data, err := os.ReadFile(configPath) if err != nil { return err } @@ -64,25 +209,34 @@ func checkUpdateTopology(configPath, releasePath string, out io.Writer) error { return fmt.Errorf("observe topology: %w", err) } sections := make(map[string]bool) - for _, section := range manifest.Required.ConfigSections { + for _, section := range required.ConfigSections { sections[section] = hasConfigSection(document, section) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - output, observeErr := exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output() - units := observedUnits(output, observeErr) - plan := manifest.Required.MigrationPlan(units, sections) - if len(plan) != 0 { - fmt.Fprintln(out, "Required migration, in order:") - for _, step := range plan { - fmt.Fprintln(out, " -", step) + units := make(map[string]bool) + var observeErr error + if managed { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + var output []byte + output, observeErr = exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output() + units = observedUnits(output, observeErr) + } else { + for _, unit := range required.Units { + units[unit] = true } - if observeErr != nil { - return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr) - } - return errors.New("refusing partial installation; apply the printed migration before retrying") } - return nil + plan := required.MigrationPlan(units, sections) + if len(plan) == 0 { + return nil + } + fmt.Fprintln(out, "Required migration, in order:") + for _, step := range plan { + fmt.Fprintln(out, " -", step) + } + if observeErr != nil { + return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr) + } + return errors.New("refusing partial installation; apply the printed migration before retrying") } func hasConfigSection(document map[string]any, section string) bool { @@ -108,10 +262,11 @@ func updateStateDir() string { return filepath.Join(dir, "archie", "update") } -// Recovery runs independently of the daemon unit: it must be able to stop that -// unit and replace its executable without systemd killing the recovery process. -func resumeUpdate(ctx context.Context) (bool, error) { - journal := filepath.Join(updateStateDir(), "transaction") +// resumeUpdate hands an interrupted transaction to the frozen watchdog and +// reports that this boot must stop: recovery may replace this executable and +// restart this unit, so it runs outside it. +func resumeUpdate(ctx context.Context, configPath string) (bool, error) { + journal := releaseupdate.JournalPath(updateStateDir()) if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) { return false, nil } else if err != nil { @@ -123,14 +278,15 @@ func resumeUpdate(ctx context.Context) (bool, error) { } defer lock.Close() if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) { + // A live watchdog owns the transaction and is restarting this unit. return false, nil } else if err != nil { return false, err } - defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }() - cmd := exec.CommandContext(ctx, "systemd-run", "--user", "--collect", "--quiet", "--unit=archied-update-recovery", "--setenv=ARCHIE_UPDATE_STATE_DIR="+updateStateDir(), filepath.Join(journal, "watchdog"), "--boot-recover") - if output, err := cmd.CombinedOutput(); err != nil { - return false, fmt.Errorf("schedule update recovery: %w: %s", err, output) + _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) + frozen := filepath.Join(journal, releaseupdate.WatchdogBinary) + if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil { + return false, fmt.Errorf("schedule update recovery: %w", err) } return true, nil } diff --git a/internal/app/archied/update_check.go b/internal/app/archied/update_check.go index 8df559bf5..f89ff2b7b 100644 --- a/internal/app/archied/update_check.go +++ b/internal/app/archied/update_check.go @@ -11,18 +11,14 @@ import ( "github.com/samcharles93/archie-core/internal/releaseupdate" ) -// RunUpdate serves `archied update check`, printing the release snapshot as -// JSON. The channel defaults to the environment the installer passes on. -func RunUpdate(args []string, out, stderr io.Writer) int { - if len(args) == 0 || args[0] != "check" { - fmt.Fprintln(stderr, "usage: archied update check [-channel stable|next|exact-pin] [-pin version]") - return 2 - } +// runUpdateCheck serves `archied update check`, printing the release snapshot +// as JSON. The channel defaults to the environment. +func runUpdateCheck(args []string, out, stderr io.Writer) int { flags := flag.NewFlagSet("update check", flag.ContinueOnError) flags.SetOutput(stderr) channel := flags.String("channel", envOr("ARCHIE_UPDATE_RELEASE_CHANNEL", "stable"), "stable, next, or exact-pin") pin := flags.String("pin", os.Getenv("ARCHIE_UPDATE_PIN"), "exact release version") - if err := flags.Parse(args[1:]); err != nil { + if err := flags.Parse(args); err != nil { return 2 } catalog := &releaseupdate.GitHubCatalog{Channel: func(context.Context) (string, string, error) { return *channel, *pin, nil }} diff --git a/internal/app/archied/update_host.go b/internal/app/archied/update_host.go new file mode 100644 index 000000000..52e461080 --- /dev/null +++ b/internal/app/archied/update_host.go @@ -0,0 +1,101 @@ +package archied + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os/exec" + "strings" + "time" + + "github.com/samcharles93/archie-core/internal/app/statestore" + "github.com/samcharles93/archie-core/internal/domain/health" +) + +// systemdUnits drives the archie user units through systemctl --user. +type systemdUnits struct{} + +// Managed reports whether a systemd user manager answers. +func (systemdUnits) Managed(ctx context.Context) bool { + return systemctl(ctx, "show-environment") == nil +} + +func (systemdUnits) Start(ctx context.Context, unit string) error { + return systemctl(ctx, "restart", unit+".service") +} + +func (systemdUnits) Stop(ctx context.Context, unit string) error { + return systemctl(ctx, "stop", unit+".service") +} + +func systemctl(ctx context.Context, args ...string) error { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + output, err := exec.CommandContext(ctx, "systemctl", append([]string{"--user"}, args...)...).CombinedOutput() + if err != nil { + return fmt.Errorf("systemctl --user %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return nil +} + +// dockerImages pulls through the Docker CLI. +type dockerImages struct{} + +func (dockerImages) Pull(ctx context.Context, ref string) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Minute) + defer cancel() + if output, err := exec.CommandContext(ctx, "docker", "pull", ref).CombinedOutput(); err != nil { + return fmt.Errorf("docker pull %s: %w: %s", ref, err, strings.TrimSpace(string(output))) + } + return nil +} + +// stateDatabase is the task database, backed up and restored in-process by +// the State Store's own recovery code. +type stateDatabase struct{ configPath string } + +func (d stateDatabase) Backup(ctx context.Context, out string) error { + _, err := statestore.RunRecovery(ctx, statestore.RecoveryOptions{Operation: statestore.RecoveryBackup, Out: out, Config: d.configPath}) + return err +} + +func (d stateDatabase) Restore(ctx context.Context, from string) error { + _, err := statestore.RunRecovery(ctx, statestore.RecoveryOptions{Operation: statestore.RecoveryRestore, From: from, Config: d.configPath}) + return err +} + +func (d stateDatabase) SchemaVersion(ctx context.Context) (int64, error) { + return statestore.SchemaVersion(ctx, statestore.RecoveryOptions{Config: d.configPath}) +} + +// daemonHealth asks archied's /health/detailed, which answers 200 only once +// the daemon serves and every readiness probe passes. +type daemonHealth struct{ url string } + +func (h daemonHealth) Healthy(ctx context.Context) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.url+"/health/detailed", nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", err + } + var report struct { + Status health.Status `json:"status"` + Version string `json:"version"` + } + if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &report) != nil || report.Status != health.StatusOK { + return report.Version, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(body))) + } + return report.Version, nil +} diff --git a/internal/app/archied/update_image.go b/internal/app/archied/update_image.go index 09d47ca92..299c5d5cc 100644 --- a/internal/app/archied/update_image.go +++ b/internal/app/archied/update_image.go @@ -5,14 +5,8 @@ import ( "crypto/sha256" "encoding/json" "errors" - "flag" "fmt" - "io" "log/slog" - "os" - "path/filepath" - "regexp" - "time" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/app/servicekit" @@ -21,34 +15,43 @@ import ( "github.com/samcharles93/archie-core/internal/secret" ) -type imageChange struct { - Previous string `json:"previous"` - Candidate string `json:"candidate"` +// imagePolicy is the control plane's agent image, reached through the State +// Store contract. Only the image field changes, preserving concurrent edits +// to unrelated settings. +type imagePolicy struct{ configPath string } + +func (p imagePolicy) Image(ctx context.Context) (string, error) { + var image string + err := p.with(ctx, func(_ pb.ControlPlaneServiceClient, _ *pb.QueryResponse, _ map[string]json.RawMessage, current string) error { + image = current + return nil + }) + return image, err } -// RunUpdateImage journals and switches the control-plane image policy. It only -// changes the image field, preserving concurrent edits to unrelated settings. -func RunUpdateImage(args []string, stderr io.Writer) int { - flags := flag.NewFlagSet("update-image", flag.ContinueOnError) - flags.SetOutput(stderr) - configPath := flags.String("config", "", "bootstrap config") - journal := flags.String("journal", "", "transaction directory") - candidate := flags.String("prepare", "", "candidate image digest") - rollback := flags.Bool("rollback", false, "restore the prior image") - if err := flags.Parse(args); err != nil { - return 1 - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := updateImage(ctx, *configPath, *journal, *candidate, *rollback); err != nil { - fmt.Fprintln(stderr, err) - return 1 - } - return 0 +func (p imagePolicy) SetImage(ctx context.Context, expected, desired string) error { + return p.with(ctx, func(rpc pb.ControlPlaneServiceClient, response *pb.QueryResponse, policy map[string]json.RawMessage, current string) error { + if current == desired { + return nil + } + if current != expected { + return errors.New("image policy changed during update; refusing to overwrite it") + } + var err error + if policy["image"], err = json.Marshal(desired); err != nil { + return err + } + data, err := json.Marshal(policy) + if err != nil { + return err + } + _, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)}) + return err + }) } -func updateImage(ctx context.Context, configPath, journal, candidate string, rollback bool) error { - _, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "") +func (p imagePolicy) with(ctx context.Context, use func(pb.ControlPlaneServiceClient, *pb.QueryResponse, map[string]json.RawMessage, string) error) error { + _, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), p.configPath, "") if err != nil { return err } @@ -70,48 +73,5 @@ func updateImage(ctx context.Context, configPath, journal, candidate string, rol if err := json.Unmarshal(policy["image"], ¤t); err != nil { return err } - path := filepath.Join(journal, "image.json") - if candidate != "" { - return prepareImageChange(path, current, candidate) - } - - data, err := os.ReadFile(path) - if err != nil { - return err - } - var change imageChange - if err := json.Unmarshal(data, &change); err != nil { - return err - } - expected, desired := change.Previous, change.Candidate - if rollback { - expected, desired = desired, expected - } - if current == desired { - return nil - } - if current != expected { - return errors.New("image policy changed during update; refusing to overwrite it") - } - policy["image"], err = json.Marshal(desired) - if err != nil { - return err - } - data, err = json.Marshal(policy) - if err != nil { - return err - } - _, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)}) - return err -} - -func prepareImageChange(path, current, candidate string) error { - if !regexp.MustCompile(`^([a-zA-Z0-9./:_-]+@)?sha256:[a-f0-9]{64}$`).MatchString(candidate) { - return errors.New("candidate image must be pinned by digest") - } - data, err := json.Marshal(imageChange{Previous: current, Candidate: candidate}) - if err != nil { - return err - } - return os.WriteFile(path, data, 0o600) + return use(rpc, response, policy, current) } diff --git a/internal/app/archied/update_test.go b/internal/app/archied/update_test.go index 54dce09b2..e86a92520 100644 --- a/internal/app/archied/update_test.go +++ b/internal/app/archied/update_test.go @@ -8,18 +8,19 @@ import ( "testing" "golang.org/x/sys/unix" + + "github.com/samcharles93/archie-core/internal/releaseupdate" ) func TestUpdateTopologyRefusal(t *testing.T) { root := t.TempDir() - manifest := filepath.Join(root, "release.json") config := filepath.Join(root, "config.toml") - writeUpdateFixture(t, manifest, `{"required_topology":{"units":["archie-state-store","archie-gateway","archied"],"config_sections":["services.state","services.gateway"]}}`) + required := releaseupdate.Topology{Units: []string{"archie-state-store", "archie-gateway", "archied"}, ConfigSections: []string{"services.state", "services.gateway"}} writeUpdateFixture(t, config, "[services.state]\ntarget='localhost:9090'\n") writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\nprintf 'archied.service enabled enabled\\n'\n") t.Setenv("PATH", root+":"+os.Getenv("PATH")) var output bytes.Buffer - if err := checkUpdateTopology(config, manifest, &output); err == nil { + if err := checkUpdateTopology(t.Context(), config, required, true, &output); err == nil { t.Fatal("partial topology accepted") } for _, missing := range []string{"configure [services.gateway]", "install and enable archie-state-store.service", "install and enable archie-gateway.service"} { @@ -39,6 +40,7 @@ func TestResumeUpdate(t *testing.T) { } called := filepath.Join(root, "called") writeUpdateFixture(t, filepath.Join(root, "systemd-run"), "#!/bin/sh\nprintf '%s\\n' \"$@\" > '"+called+"'\n") + writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\n") t.Setenv("PATH", root+":"+os.Getenv("PATH")) lock, err := os.OpenFile(filepath.Join(root, "lock"), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { @@ -50,7 +52,7 @@ func TestResumeUpdate(t *testing.T) { t.Fatal(err) } } - resumed, err := resumeUpdate(t.Context()) + resumed, err := resumeUpdate(t.Context(), filepath.Join(root, "config.toml")) if err != nil || resumed == active { t.Fatalf("resumed=%v err=%v", resumed, err) } @@ -59,7 +61,7 @@ func TestResumeUpdate(t *testing.T) { if err != nil { t.Fatal(err) } - if !strings.Contains(string(data), "--boot-recover") { + if !strings.Contains(string(data), "-recover") { t.Fatalf("not a rollback: %s", data) } } diff --git a/internal/app/archiegateway/chat_setup.go b/internal/app/archiegateway/chat_setup.go index 909fd123f..f5cd23449 100644 --- a/internal/app/archiegateway/chat_setup.go +++ b/internal/app/archiegateway/chat_setup.go @@ -100,7 +100,7 @@ func componentInstallTypeEnricher(nats config.NATSConfig) func(string) (string, } func makeUpdateService(reader controlplanerpc.ResourceReader, cfg config.Config) *releaseupdate.Service { - updates := controlplanerpc.UpdateService(reader, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"), cfg.Health.URL()) + updates := controlplanerpc.UpdateService(reader, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json")) updates.Enrich = componentInstallTypeEnricher(cfg.NATS) return updates } diff --git a/internal/app/archiemessaging/config.go b/internal/app/archiemessaging/config.go index 4df7fbb5d..f1b973f73 100644 --- a/internal/app/archiemessaging/config.go +++ b/internal/app/archiemessaging/config.go @@ -17,13 +17,10 @@ type ResolvedConfig struct { Options Options TelegramToken string Telegram config.TelegramConfig - // WorkDir, BotUser and HealthURL are not channel transport settings. They - // locate this identity's release-announcement and update-report state - // files, and give the update installer the daemon health endpoint to poll - // after it applies one. + // WorkDir and BotUser are not channel transport settings. They locate + // this identity's release-announcement and update-report state files. WorkDir string BotUser string - HealthURL string ShowToolCalls bool } @@ -34,7 +31,6 @@ type projection struct { telegram config.TelegramConfig workDir string botUser string - healthURL string showToolCalls bool } @@ -65,7 +61,6 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) { Telegram: proj.telegram, WorkDir: proj.workDir, BotUser: proj.botUser, - HealthURL: proj.healthURL, ShowToolCalls: proj.showToolCalls, }, nil } @@ -92,7 +87,6 @@ func project(cfg config.Config) projection { telegram: cfg.Chat.Telegram, workDir: cfg.WorkDir, botUser: cfg.BotUser, - healthURL: cfg.Health.URL(), showToolCalls: cfg.Chat.ShowToolCalls, } } diff --git a/internal/app/archiemessaging/run.go b/internal/app/archiemessaging/run.go index 2dbe9788a..4656a5452 100644 --- a/internal/app/archiemessaging/run.go +++ b/internal/app/archiemessaging/run.go @@ -86,7 +86,7 @@ func Run(ctx context.Context, o Options) error { settings = messaging.NewSettingsCommand(controlplanerpc.NewSettingsClient(stateStore.ControlPlane())).WithIdentities(stateStore) channelStatusStore, presenceStore = stateStore, stateStore settingsSource, applyReporter, appliedVersion = controlPlaneClient, reporter, channelVersion - updates = controlplanerpc.UpdateService(controlPlaneClient, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"), cfg.HealthURL) + updates = controlplanerpc.UpdateService(controlPlaneClient, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json")) } if cfg.Options.StateStore.Target == "" { diff --git a/internal/app/archieui/run.go b/internal/app/archieui/run.go index 07fc0af99..01621fed1 100644 --- a/internal/app/archieui/run.go +++ b/internal/app/archieui/run.go @@ -77,7 +77,7 @@ func Run(ctx context.Context, options Options) error { Log: log, Store: tasks, Chat: chat, - Updates: controlplanerpc.UpdateService(controlplanerpc.NewRPCClient(tasks.ControlPlane()), filepath.Join(configuration.DefaultWorkDir(), "ui-update-deferrals.json"), ""), + Updates: controlplanerpc.UpdateService(controlplanerpc.NewRPCClient(tasks.ControlPlane()), filepath.Join(configuration.DefaultWorkDir(), "ui-update-deferrals.json")), Health: newReadinessRegistry(opts, tasks, chat, chain, provider), ControlPlane: tasks.ControlPlane(), Identities: tasks, diff --git a/internal/app/statestore/recovery_postgres.go b/internal/app/statestore/recovery_postgres.go index 75e3b9522..7ba6f5f5d 100644 --- a/internal/app/statestore/recovery_postgres.go +++ b/internal/app/statestore/recovery_postgres.go @@ -107,3 +107,22 @@ func rollbackPostgres(ctx context.Context, url string, options RecoveryOptions) defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }() return replayRevision(ctx, postgres.NewResources(pool), options) } + +// SchemaVersion reads the database's applied migration without migrating or +// refusing a schema newer than this binary, so an older binary can tell that +// a newer one migrated it. +func SchemaVersion(ctx context.Context, options RecoveryOptions) (int64, error) { + cfg, err := bootConfig(ctx, options) + if err != nil { + return 0, err + } + if cfg.DatabaseURL == "" { + return 0, errors.New("database_url is required: the configuration names no PostgreSQL database") + } + pool, err := postgres.Open(ctx, cfg.DatabaseURL) + if err != nil { + return 0, err + } + defer pool.Close() + return postgres.SchemaVersion(ctx, pool) +} diff --git a/internal/channels/telegram/update.go b/internal/channels/telegram/update.go index 401137c85..f36e400bd 100644 --- a/internal/channels/telegram/update.go +++ b/internal/channels/telegram/update.go @@ -271,7 +271,11 @@ func formatPendingReport(report releaseupdate.Report, running map[string]string) previousVersions := strings.Join(versions, ", ") if report.RolledBack { - return fmt.Sprintf("Update failed health check after restarting and was automatically rolled back to the previous version (%s). archie-core is running normally on the previous release.", previousVersions) + text := fmt.Sprintf("Update failed health check after restarting and was automatically rolled back to the previous version (%s). archie-core is running normally on the previous release.", previousVersions) + if report.RestoredDatabase { + text += " The update had migrated the database, so the pre-update backup was restored: every write made since the update began is gone." + } + return text } return fmt.Sprintf("Update failed health check after restarting and could NOT be rolled back automatically (no backup was found). archie-core may still be running the broken version, or may not be running at all — manual intervention is required. Previous version was %s.", previousVersions) } diff --git a/internal/config/config.go b/internal/config/config.go index 28fbd39bf..dda061c7b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -916,7 +916,7 @@ type Health struct { } // URL renders Listen as an address a local caller can dial, which is what -// the daemon hands its update tooling (see releaseupdate.CommandInstaller). +// the update watchdog polls after restarting the daemon. // A wildcard bind is a valid thing to listen on but not to dial, so it // resolves to localhost, the same substitution the dashboard link makes. func (h Health) URL() string { diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index 6553dced9..699c064ab 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -41,8 +41,7 @@ const ( defaultCaptureRatePerSecond = 1.0 defaultCaptureRateBurst = 5 - // defaultHealthListen is the daemon's own liveness address. The update - // watchdog's fallback in scripts/archie-update-watchdog must match it. + // defaultHealthListen is the daemon's own liveness address. defaultHealthListen = "127.0.0.1:8485" // defaultHealthDependencyTimeout matches the extracted services' diff --git a/internal/infrastructure/controlplanerpc/vocabulary.go b/internal/infrastructure/controlplanerpc/vocabulary.go index ccb09cc0a..ecf6da530 100644 --- a/internal/infrastructure/controlplanerpc/vocabulary.go +++ b/internal/infrastructure/controlplanerpc/vocabulary.go @@ -118,12 +118,12 @@ type UpdateSettings struct { } // UpdateService checks GitHub on the stored release channel and installs -// through the script beside this binary. statePath keeps each recipient's -// deferrals; healthURL reaches the installer. -func UpdateService(reader ResourceReader, statePath, healthURL string) *releaseupdate.Service { +// through the archied beside this binary. statePath keeps each recipient's +// deferrals. +func UpdateService(reader ResourceReader, statePath string) *releaseupdate.Service { return &releaseupdate.Service{ Catalog: &releaseupdate.GitHubCatalog{Channel: updateChannel(reader)}, - Installer: releaseupdate.ScriptInstaller{HealthURL: healthURL}, + Installer: releaseupdate.ArchiedInstaller{}, StatePath: statePath, InstallType: installtype.Type(), } diff --git a/internal/releaseupdate/command.go b/internal/releaseupdate/command.go deleted file mode 100644 index 52ae9088d..000000000 --- a/internal/releaseupdate/command.go +++ /dev/null @@ -1,141 +0,0 @@ -package releaseupdate - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" -) - -// updateResultSentinel prefixes the one line an install command may print to -// report its structured Result. Every other line is forwarded to progress -// as-is. -const updateResultSentinel = "ARCHIE_UPDATE_RESULT " - -// CommandInstaller runs the deployment's install command. HealthURL is -// passed as ARCHIE_HEALTH_URL when set. -type CommandInstaller struct { - Command []string - HealthURL string -} - -// Install runs the command, streaming stdout lines to progress. One line may -// be a structured Result. -func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) { - if len(i.Command) == 0 { - return Result{}, fmt.Errorf("update install command is empty") - } - cmd := exec.CommandContext(ctx, i.Command[0], i.Command[1:]...) - versions := componentVersions(snapshot) - cmd.Env = append( - cmd.Environ(), - "ARCHIE_UPDATE_CHANNEL="+meta.Channel, - "ARCHIE_UPDATE_CHAT_ID="+strconv.FormatInt(meta.ChatID, 10), - "ARCHIE_UPDATE_THREAD_ID="+strconv.Itoa(meta.ThreadID), - "ARCHIE_UPDATE_REPORT_PATH="+meta.ReportPath, - "ARCHIE_UPDATE_DAEMON_PREVIOUS="+versions[ComponentDaemon].Installed, - "ARCHIE_UPDATE_DAEMON_VERSION="+versions[ComponentDaemon].Available, - "ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed, - "ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available, - ) - if i.HealthURL != "" { - cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL) - } - - stdout, err := cmd.StdoutPipe() - if err != nil { - return Result{}, fmt.Errorf("open update installer stdout: %w", err) - } - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Start(); err != nil { - return Result{}, fmt.Errorf("start update installer: %w", err) - } - - var result Result - var resultErr error - scanner := bufio.NewScanner(stdout) - scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) - for scanner.Scan() { - line := scanner.Text() - if rest, ok := strings.CutPrefix(line, updateResultSentinel); ok { - if err := json.Unmarshal([]byte(rest), &result); err != nil { - resultErr = fmt.Errorf("decode update installer result: %w", err) - } - continue - } - progress(line) - } - if err := scanner.Err(); err != nil { - resultErr = fmt.Errorf("read update installer output: %w", err) - } - - waitErr := cmd.Wait() - if waitErr != nil { - return result, fmt.Errorf("run update installer: %w: %s", waitErr, stderr.String()) - } - if resultErr != nil { - return result, resultErr - } - return result, nil -} - -// componentVersions reduces an approved snapshot to the two components the -// reference binary installer owns. Available is intentionally blank for an -// unchanged component: the install script treats these values as the complete -// approved plan, not as hints to rediscover work after approval. -func componentVersions(snapshot Snapshot) map[string]Component { - versions := map[string]Component{ - ComponentDaemon: {}, - ComponentAgent: {}, - } - for _, component := range snapshot.Components { - if component.ID != ComponentDaemon && component.ID != ComponentAgent { - continue - } - if component.Available == component.Installed { - component.Available = "" - } - versions[component.ID] = component - } - return versions -} - -// ErrNotConfigured is returned when the step's command is unset. -var ErrNotConfigured = errors.New("updates are not configured") - -// ScriptInstaller runs the install script a release places beside this binary. -type ScriptInstaller struct { - HealthURL string -} - -func (s ScriptInstaller) path() string { - exe, err := os.Executable() - if err != nil { - return "" - } - path := filepath.Join(filepath.Dir(exe), "archie-update-install") - if info, err := os.Stat(path); err != nil || info.Mode()&0o111 == 0 { - return "" - } - return path -} - -func (s ScriptInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) { - path := s.path() - if path == "" { - return Result{}, ErrNotConfigured - } - return CommandInstaller{Command: []string{path}, HealthURL: s.HealthURL}.Install(ctx, snapshot, meta, progress) -} - -// Installable reports whether the install script is present. -func (s ScriptInstaller) Installable(context.Context) bool { return s.path() != "" } diff --git a/internal/releaseupdate/github.go b/internal/releaseupdate/github.go index 9a3aff253..d1d477d57 100644 --- a/internal/releaseupdate/github.go +++ b/internal/releaseupdate/github.go @@ -64,7 +64,7 @@ func (c *GitHubCatalog) Check(ctx context.Context) (Snapshot, error) { } return Snapshot{Components: []Component{ component("daemon", "Gateway", installedVersion(buildinfo.Version)), - component("agent", "Runtime", installedAgent()), + component("agent", "Runtime", installedAgent(executableDir())), }}, nil } @@ -197,13 +197,9 @@ func installedVersion(version string) string { // installedAgent is the image the installer last switched to, recorded in a // sidecar beside the binaries; without one it is the runtime this release was // cut with. A malformed sidecar is unknown, not guessed. -func installedAgent() string { - exe, err := os.Executable() - if err != nil { - return installedVersion(buildinfo.Runtime) - } - data, err := os.ReadFile(filepath.Join(filepath.Dir(exe), "archie-agent.version")) - if errors.Is(err, os.ErrNotExist) { +func installedAgent(binDir string) string { + data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile)) + if binDir == "" || errors.Is(err, os.ErrNotExist) { return installedVersion(buildinfo.Runtime) } version := strings.TrimSpace(string(data)) @@ -212,3 +208,11 @@ func installedAgent() string { } return installedVersion(version) } + +func executableDir() string { + exe, err := os.Executable() + if err != nil { + return "" + } + return filepath.Dir(exe) +} diff --git a/internal/releaseupdate/installer.go b/internal/releaseupdate/installer.go new file mode 100644 index 000000000..0ffa3056e --- /dev/null +++ b/internal/releaseupdate/installer.go @@ -0,0 +1,86 @@ +package releaseupdate + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" +) + +// ResultSentinel prefixes the one line `archied update install` prints to +// report its Result. Every other line is progress. +const ResultSentinel = "ARCHIE_UPDATE_RESULT " + +// ErrNotConfigured is returned when no archied sits beside this binary to +// install with. +var ErrNotConfigured = errors.New("updates are not configured") + +// ArchiedInstaller runs `archied update install` from beside this binary, so +// every service installs through the daemon's one transaction. +type ArchiedInstaller struct{} + +func (ArchiedInstaller) path() string { + path := filepath.Join(executableDir(), WatchdogBinary) + if info, err := os.Stat(path); err != nil || info.Mode()&0o111 == 0 { + return "" + } + return path +} + +// Installable reports whether archied is present to install with. +func (i ArchiedInstaller) Installable(context.Context) bool { return i.path() != "" } + +// Install installs the snapshot's release, streaming progress lines. +func (i ArchiedInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) { + path := i.path() + if path == "" { + return Result{}, ErrNotConfigured + } + available := snapshot.Available() + if len(available) == 0 { + return Result{}, errors.New("nothing to install") + } + cmd := exec.CommandContext(ctx, path, "update", "install", + "-version", available[0].Available, + "-report", meta.ReportPath, + "-channel", meta.Channel, + "-chat", strconv.FormatInt(meta.ChatID, 10), + "-thread", strconv.Itoa(meta.ThreadID)) + stdout, err := cmd.StdoutPipe() + if err != nil { + return Result{}, fmt.Errorf("open update installer stdout: %w", err) + } + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + return Result{}, fmt.Errorf("start update installer: %w", err) + } + var result Result + var resultErr error + scanner := bufio.NewScanner(stdout) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Text() + if rest, ok := strings.CutPrefix(line, ResultSentinel); ok { + if err := json.Unmarshal([]byte(rest), &result); err != nil { + resultErr = fmt.Errorf("decode update installer result: %w", err) + } + continue + } + progress(line) + } + if err := scanner.Err(); err != nil { + resultErr = fmt.Errorf("read update installer output: %w", err) + } + if err := cmd.Wait(); err != nil { + return result, fmt.Errorf("run update installer: %w: %s", err, strings.TrimSpace(stderr.String())) + } + return result, resultErr +} diff --git a/internal/releaseupdate/installer_test.go b/internal/releaseupdate/installer_test.go deleted file mode 100644 index 1d32a0ddc..000000000 --- a/internal/releaseupdate/installer_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package releaseupdate - -import ( - "archive/zip" - "crypto/sha256" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - "golang.org/x/sys/unix" -) - -func TestUpdateTransaction(t *testing.T) { - for _, scenario := range []string{"healthy", "checksum", "unsafe-path", "missing-digest", "busy", "lock", "unhealthy", "crash"} { - t.Run(scenario, func(t *testing.T) { - f := newUpdateFixture(t, scenario) - var lock *os.File - if scenario == "lock" { - var err error - lock, err = os.OpenFile(filepath.Join(f.state, "lock"), os.O_CREATE|os.O_RDWR, 0o600) - if err != nil { - t.Fatal(err) - } - defer lock.Close() - if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { - t.Fatal(err) - } - fixtureWrite(t, filepath.Join(f.state, "holder"), "fixture-owner", 0o600) - } - args := []string{} - if scenario == "busy" { - args = append(args, "--auto") - } - output, err := f.run(t, f.installer, args...) - refusal := map[string]string{"checksum": "checksum mismatch", "unsafe-path": "unsafe archive path", "missing-digest": "digest-pinned", "busy": "work in flight", "lock": "fixture-owner"}[scenario] - if refusal != "" { - if err == nil || !strings.Contains(output, refusal) { - t.Fatalf("wanted %q refusal; err=%v output=%s", refusal, err, output) - } - f.assertVersion(t, "old") - return - } - if err != nil { - t.Fatalf("installer: %v\n%s", err, output) - } - f.assertVersion(t, "old") // Staging cannot mutate a running executable. - old, err := os.Open(filepath.Join(f.bin, "archied")) - if err != nil { - t.Fatal(err) - } - defer old.Close() - watchdog := filepath.Join(f.state, "transaction", "watchdog") - output, err = f.run(t, watchdog) - if scenario == "crash" { - if err == nil { - t.Fatal("expected injected crash") - } - f.env = append(f.env, "CRASH_AFTER=") - output, err = f.run(t, watchdog, "--boot-recover") - } - if err != nil { - t.Fatalf("watchdog: %v\n%s", err, output) - } - want := "old" - if scenario == "healthy" { - want = "new" - } - f.assertVersion(t, want) - data := make([]byte, 100) - n, _ := old.Read(data) - if !strings.Contains(string(data[:n]), "old") { - t.Fatal("promotion overwrote the open old inode") - } - if _, err := os.Stat(filepath.Join(f.state, "transaction")); !os.IsNotExist(err) { - t.Fatalf("journal not retired: %v", err) - } - }) - } -} - -type updateFixture struct { - bin, state, installer string - env []string -} - -func fixtureWrite(t *testing.T, path, data string, mode os.FileMode) { - t.Helper() - if err := os.WriteFile(path, []byte(data), mode); err != nil { - t.Fatal(err) - } -} - -func newUpdateFixture(t *testing.T, scenario string) updateFixture { - t.Helper() - root := t.TempDir() - bin, state, fake := filepath.Join(root, "bin"), filepath.Join(root, "state"), filepath.Join(root, "fake") - for _, dir := range []string{bin, state, fake} { - if err := os.MkdirAll(dir, 0o700); err != nil { - t.Fatal(err) - } - } - scripts, err := filepath.Abs("../../scripts") - if err != nil { - t.Fatal(err) - } - for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} { - fixtureWrite(t, filepath.Join(bin, name), "#!/bin/bash\n# old\nexit 0\n", 0o755) - } - config := filepath.Join(root, "config.toml") - fixtureWrite(t, config, "[services.state]\n[services.gateway]\n", 0o600) - archive := makeUpdateArchive(t, root, scripts, scenario) - sum := sha256.Sum256(archive) - if scenario == "checksum" { - sum[0]++ - } - fixtureWrite(t, filepath.Join(root, "SHA256SUMS"), fmt.Sprintf("%x archie-core-archied-v2.0.0-linux-amd64.zip\n", sum), 0o600) - fixtureWrite(t, filepath.Join(fake, "curl"), `#!/bin/bash -out="" -for ((i=1;i<=$#;i++)); do - if [ "${!i}" = -o ]; then j=$((i+1)); out="${!j}"; fi - case "${!i}" in - */SHA256SUMS) source="$FIXTURE/SHA256SUMS" ;; - *.zip) source="$FIXTURE/archive.zip" ;; - esac -done -if [ -n "$out" ]; then cp "$source" "$out"; exit; fi -if [ "$SCENARIO" = unhealthy ]; then exit 22; fi -busy=0 -[ "$SCENARIO" = busy ] && busy=1 -printf '{"status":"ok","version":"2.0.0","in_flight":%s}\n' "$busy" -`, 0o755) - for _, name := range []string{"systemctl", "systemd-run", "docker"} { - fixtureWrite(t, filepath.Join(fake, name), "#!/bin/bash\nexit 0\n", 0o755) - } - fixtureWrite(t, filepath.Join(fake, "git"), "#!/bin/bash\necho unexpected-git >&2\nexit 98\n", 0o755) - fixtureWrite(t, filepath.Join(fake, "mv"), `#!/bin/bash -/bin/mv "$@" || exit -if [ -n "$CRASH_AFTER" ] && [[ "$*" == *".archied.new"* ]]; then kill -KILL "$PPID"; fi -`, 0o755) - return updateFixture{bin: bin, state: state, installer: filepath.Join(scripts, "archie-update-install"), env: append(os.Environ(), - "PATH="+fake+":"+os.Getenv("PATH"), "FIXTURE="+root, "SCENARIO="+scenario, - "ARCHIE_BIN_DIR="+bin, "ARCHIE_CONFIG_PATH="+config, "ARCHIE_UPDATE_STATE_DIR="+state, - "ARCHIE_UPDATE_DAEMON_PREVIOUS=1.0.0", "ARCHIE_UPDATE_DAEMON_VERSION=2.0.0", - "ARCHIE_UPDATE_AGENT_PREVIOUS=1.0.0", "ARCHIE_UPDATE_AGENT_VERSION="+map[bool]string{true: "2.0.0"}[scenario == "missing-digest"], - "ARCHIE_UPDATE_HEALTH_TIMEOUT=1", "ARCHIE_UPDATE_OBSERVE_SECONDS=0", "ARCHIE_UPDATE_POLL_INTERVAL=0.01", - "ARCHIE_UPDATE_STATE_STORE_TARGET=", "CRASH_AFTER="+map[bool]string{true: "archied"}[scenario == "crash"], - )} -} - -func makeUpdateArchive(t *testing.T, root, scripts, scenario string) []byte { - t.Helper() - file, err := os.Create(filepath.Join(root, "archive.zip")) - if err != nil { - t.Fatal(err) - } - z := zip.NewWriter(file) - files := map[string]string{"release.json": `{"required_topology":{"units":["archie-state-store","archie-gateway","archie-ui","archie-messaging","archied"],"config_sections":["services.state","services.gateway"]},"agent_image":"mutable:latest"}`} - for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} { - files[name] = "#!/bin/bash\n# new\nif [ \"$1\" = -version ]; then echo '" + name + " 2.0.0'; fi\nexit 0\n" - } - for _, name := range []string{"archie-update-install", "archie-update-watchdog"} { - data, err := os.ReadFile(filepath.Join(scripts, name)) - if err != nil { - t.Fatal(err) - } - files[name] = string(data) - } - if scenario == "unsafe-path" { - files["../escape"] = "escape" - } - for name, body := range files { - header := &zip.FileHeader{Name: "archie-core-archied-v2.0.0-linux-amd64/" + name, Method: zip.Deflate} - header.SetMode(0o755) - w, err := z.CreateHeader(header) - if err != nil { - t.Fatal(err) - } - if _, err := w.Write([]byte(body)); err != nil { - t.Fatal(err) - } - } - if err := z.Close(); err != nil { - t.Fatal(err) - } - if err := file.Close(); err != nil { - t.Fatal(err) - } - data, err := os.ReadFile(file.Name()) - if err != nil { - t.Fatal(err) - } - return data -} - -func (f updateFixture) run(t *testing.T, path string, args ...string) (string, error) { - t.Helper() - cmd := exec.CommandContext(t.Context(), path, args...) - cmd.Env = f.env - data, err := cmd.CombinedOutput() - return string(data), err -} - -func (f updateFixture) assertVersion(t *testing.T, want string) { - t.Helper() - data, err := os.ReadFile(filepath.Join(f.bin, "archied")) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(data), "# "+want) { - t.Fatalf("wanted %s executable: %s", want, data) - } -} diff --git a/internal/releaseupdate/pending_report.go b/internal/releaseupdate/pending_report.go index 36db9c668..898e35661 100644 --- a/internal/releaseupdate/pending_report.go +++ b/internal/releaseupdate/pending_report.go @@ -7,8 +7,8 @@ import ( "os" ) -// Report is an update's post-restart outcome, written by the restart tooling -// and relayed by the next archied boot. +// Report is an update's post-restart outcome, written by the watchdog and +// relayed by the next archied boot. type Report struct { Channel string `json:"channel"` ChatID int64 `json:"chat_id"` @@ -17,6 +17,9 @@ type Report struct { Installed map[string]string `json:"installed"` HealthCheck string `json:"health_check"` // "passed" or "failed" RolledBack bool `json:"rolled_back"` + // RestoredDatabase reports that rollback found a migration and restored + // the pre-update snapshot, losing every write made since it. + RestoredDatabase bool `json:"restored_database"` } func (r Report) validate() error { diff --git a/internal/releaseupdate/prepare.go b/internal/releaseupdate/prepare.go new file mode 100644 index 000000000..65e93e920 --- /dev/null +++ b/internal/releaseupdate/prepare.go @@ -0,0 +1,369 @@ +package releaseupdate + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/samcharles93/archie-core/internal/buildinfo" +) + +// ReleaseURL is where published release assets are downloaded from. +const ReleaseURL = "https://github.com/" + ReleaseSlug + "/releases/download" + +var ( + releaseVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$`) + releaseUnit = regexp.MustCompile(`^archie[a-z0-9-]*$`) + pinnedImage = regexp.MustCompile(`^[a-zA-Z0-9./:_-]+@sha256:[a-f0-9]{64}$`) +) + +// PrepareRequest names the release to stage and who hears the outcome. +type PrepareRequest struct { + Version string + // ReleaseURL is the base the release's zip and SHA256SUMS are under. + ReleaseURL string + // Self is the archied executable that will run the watchdog. + Self string + Meta InstallMeta + // CheckTopology refuses a host missing a unit or config section the + // release requires. managed reports whether units are observable. + CheckTopology func(ctx context.Context, required Topology, managed bool) error +} + +// Manifest is the release.json a release zip carries. +type Manifest struct { + Required Topology `json:"required_topology"` + AgentImage string `json:"agent_image"` +} + +// Prepare verifies and stages a release without changing anything that runs: +// the zip is checked against SHA256SUMS, every binary must report the release, +// the agent image is pulled by digest and the database is backed up before a +// journal is published. The watchdog (Run) takes it from there. +func (t *Transaction) Prepare(ctx context.Context, req PrepareRequest) (Result, error) { + if !releaseVersion.MatchString(req.Version) { + return Result{}, fmt.Errorf("refusing non-release version %q", req.Version) + } + unlock, held, err := lockState(ctx, t.StateDir, false) + if err != nil { + return Result{}, err + } + if !held { + return Result{}, ErrInstallInProgress + } + defer unlock() + if _, err := os.Stat(JournalPath(t.StateDir)); err == nil { + return Result{}, fmt.Errorf("an unfinished update is pending recovery at %s", JournalPath(t.StateDir)) + } + work, err := os.MkdirTemp(t.StateDir, ".release-*") + if err != nil { + return Result{}, err + } + defer func() { _ = os.RemoveAll(work) }() + + release, manifest, err := t.fetch(ctx, req, work) + if err != nil { + return Result{}, err + } + managed := t.Units.Managed(ctx) + if req.CheckTopology != nil { + if err := req.CheckTopology(ctx, manifest.Required, managed); err != nil { + return Result{}, err + } + } + j, err := t.snapshotHost(ctx, req, manifest, managed) + if err != nil { + return Result{}, err + } + if err := t.stage(req, release, j); err != nil { + for _, unit := range j.Units { + _ = os.Remove(t.staged(unit)) + } + return Result{}, err + } + installed := map[string]string{ComponentDaemon: req.Version, ComponentAgent: req.Version} + return Result{ + Previous: j.Previous, + Installed: installed, + Components: []ComponentResult{ + {ID: ComponentDaemon, Label: "Gateway", Status: "updated"}, + {ID: ComponentAgent, Label: "Runtime", Status: "updated"}, + }, + RestartRequested: managed, + }, nil +} + +// fetch downloads, verifies and unpacks the release, and checks every binary +// reports the version being installed. +func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string) (string, Manifest, error) { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + return "", Manifest{}, fmt.Errorf("no published release for %s/%s", runtime.GOOS, runtime.GOARCH) + } + name := fmt.Sprintf("archie-core-archied-v%s-linux-amd64", req.Version) + base := strings.TrimSuffix(req.ReleaseURL, "/") + "/v" + req.Version + "/" + t.say("downloading %s", name+".zip") + archive := filepath.Join(work, name+".zip") + if err := downloadVerified(ctx, base, name+".zip", archive); err != nil { + return "", Manifest{}, err + } + if err := unpack(archive, name, work); err != nil { + return "", Manifest{}, err + } + release := filepath.Join(work, name) + manifest, err := readManifest(filepath.Join(release, "release.json")) + if err != nil { + return "", Manifest{}, err + } + for _, unit := range manifest.Required.Units { + reported, err := reportedVersion(ctx, filepath.Join(release, unit), unit) + if err != nil { + return "", Manifest{}, err + } + if !sameVersion(reported, req.Version) { + return "", Manifest{}, fmt.Errorf("candidate %s reports %q, not %s", unit, reported, req.Version) + } + } + return release, manifest, nil +} + +// snapshotHost pulls the image and records what rollback needs: the current +// versions, the current image policy, the schema version and a backup. +func (t *Transaction) snapshotHost(ctx context.Context, req PrepareRequest, manifest Manifest, managed bool) (journal, error) { + t.say("pulling %s", manifest.AgentImage) + if err := t.Images.Pull(ctx, manifest.AgentImage); err != nil { + return journal{}, fmt.Errorf("pull agent image: %w", err) + } + image, err := t.Policy.Image(ctx) + if err != nil { + return journal{}, fmt.Errorf("read image policy: %w", err) + } + schema, err := t.Database.SchemaVersion(ctx) + if err != nil { + return journal{}, fmt.Errorf("read schema version: %w", err) + } + t.say("backing up the database to %s", t.snapshot()) + if err := t.Database.Backup(ctx, t.snapshot()); err != nil { + return journal{}, fmt.Errorf("back up the database: %w", err) + } + return journal{ + Phase: phasePrepared, + Version: req.Version, + Previous: map[string]string{ + ComponentDaemon: installedVersion(buildinfo.Version), + ComponentAgent: installedAgent(t.BinDir), + }, + Units: manifest.Required.Units, + Managed: managed, + Image: imageChange{Previous: image, Candidate: manifest.AgentImage}, + SchemaBefore: schema, + Meta: req.Meta, + }, nil +} + +// stage freezes the watchdog and every file rollback restores, places the +// candidates beside the live binaries, and publishes the journal last. +func (t *Transaction) stage(req PrepareRequest, release string, j journal) error { + preparing := filepath.Join(t.StateDir, "preparing") + if err := os.RemoveAll(preparing); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil { + return err + } + if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil { + return fmt.Errorf("freeze the watchdog: %w", err) + } + for _, unit := range j.Units { + live := filepath.Join(t.BinDir, unit) + if _, err := os.Stat(live); err == nil { + if err := installFile(live, filepath.Join(preparing, "previous", unit), 0o755); err != nil { + return fmt.Errorf("save %s: %w", unit, err) + } + } + if err := installFile(filepath.Join(release, unit), t.staged(unit), 0o755); err != nil { + return fmt.Errorf("stage %s: %w", unit, err) + } + } + version := filepath.Join(t.BinDir, agentVersionFile) + if _, err := os.Stat(version); err == nil { + if err := installFile(version, filepath.Join(preparing, agentVersionFile), 0o644); err != nil { + return err + } + } + if err := syncDir(t.BinDir); err != nil { + return err + } + if err := writeJournal(preparing, j); err != nil { + return err + } + if err := syncDir(filepath.Join(preparing, "previous")); err != nil { + return err + } + if err := os.Rename(preparing, JournalPath(t.StateDir)); err != nil { + return err + } + return syncDir(t.StateDir) +} + +func downloadVerified(ctx context.Context, base, name, dst string) error { + var sums bytes.Buffer + if err := download(ctx, base+"SHA256SUMS", &sums, 1<<20); err != nil { + return err + } + expected := "" + scanner := bufio.NewScanner(&sums) + for scanner.Scan() { + if fields := strings.Fields(scanner.Text()); len(fields) == 2 && fields[1] == name { + if expected != "" { + return fmt.Errorf("SHA256SUMS lists %s twice", name) + } + expected = strings.ToLower(fields[0]) + } + } + if len(expected) != sha256.Size*2 { + return fmt.Errorf("SHA256SUMS has no checksum for %s", name) + } + file, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + hash := sha256.New() + err = download(ctx, base+name, io.MultiWriter(file, hash), 1<<30) + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + if hex.EncodeToString(hash.Sum(nil)) != expected { + return fmt.Errorf("release checksum mismatch for %s", name) + } + return nil +} + +func download(ctx context.Context, url string, out io.Writer, limit int64) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download %s: %s", url, resp.Status) + } + _, err = io.Copy(out, io.LimitReader(resp.Body, limit)) + return err +} + +// unpack extracts the zip's single distribution directory, refusing any +// entry outside it and anything but regular files and directories. +func unpack(archive, name, dst string) error { + reader, err := zip.OpenReader(archive) + if err != nil { + return err + } + defer func() { _ = reader.Close() }() + for _, file := range reader.File { + clean := path.Clean(file.Name) + if !strings.HasPrefix(file.Name, name+"/") || strings.Contains(file.Name, `\`) || + clean != strings.TrimSuffix(file.Name, "/") || !strings.HasPrefix(clean, name) { + return fmt.Errorf("unsafe archive path %q", file.Name) + } + mode := file.Mode() + target := filepath.Join(dst, filepath.FromSlash(clean)) + switch { + case mode.IsDir(): + if err := os.MkdirAll(target, 0o700); err != nil { + return err + } + case mode.IsRegular(): + if err := extract(file, target); err != nil { + return err + } + default: + return fmt.Errorf("archive entry %q is not a regular file", file.Name) + } + } + return nil +} + +func extract(file *zip.File, target string) error { + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return err + } + src, err := file.Open() + if err != nil { + return err + } + defer func() { _ = src.Close() }() + dst, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700) + if err != nil { + return err + } + if _, err := io.Copy(dst, io.LimitReader(src, 1<<30)); err != nil { + _ = dst.Close() + return err + } + return dst.Close() +} + +func readManifest(path string) (Manifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return Manifest{}, err + } + var manifest Manifest + if err := json.Unmarshal(data, &manifest); err != nil { + return Manifest{}, fmt.Errorf("decode release.json: %w", err) + } + if len(manifest.Required.Units) == 0 { + return Manifest{}, errors.New("release has no required_topology") + } + for _, unit := range manifest.Required.Units { + if !releaseUnit.MatchString(unit) { + return Manifest{}, fmt.Errorf("invalid release unit %q", unit) + } + } + if !pinnedImage.MatchString(manifest.AgentImage) { + return Manifest{}, errors.New("release has no digest-pinned agent image") + } + return manifest, nil +} + +// reportedVersion runs binary -version and returns the version on the line +// naming unit. +func reportedVersion(ctx context.Context, binary, unit string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + output, err := exec.CommandContext(ctx, binary, "-version").Output() + if err != nil { + return "", fmt.Errorf("%s -version: %w", unit, err) + } + for line := range strings.SplitSeq(string(output), "\n") { + if fields := strings.Fields(line); len(fields) == 2 && fields[0] == unit { + return fields[1], nil + } + } + return "", fmt.Errorf("%s -version names no version", unit) +} diff --git a/internal/releaseupdate/report_event.go b/internal/releaseupdate/report_event.go index 25327ef09..f28df2181 100644 --- a/internal/releaseupdate/report_event.go +++ b/internal/releaseupdate/report_event.go @@ -24,13 +24,14 @@ func ReportEvent(report Report, running map[string]string) events.Event { Kind: events.KindUpdateReport, Detail: summarizeReport(report, verification), Data: map[string]any{ - "health_check": report.HealthCheck, - "rolled_back": report.RolledBack, - "previous": report.Previous, - "installed": report.Installed, - "confirmed": verification.Confirmed, - "drift": drift, - "unverified": verification.Unverified, + "health_check": report.HealthCheck, + "rolled_back": report.RolledBack, + "restored_database": report.RestoredDatabase, + "previous": report.Previous, + "installed": report.Installed, + "confirmed": verification.Confirmed, + "drift": drift, + "unverified": verification.Unverified, }, } } @@ -38,6 +39,9 @@ func ReportEvent(report Report, running map[string]string) events.Event { // summarizeReport returns a one-line summary: confirmed, drifted, unverified // or rolled back. func summarizeReport(report Report, verification Verification) string { + if report.RolledBack && report.RestoredDatabase { + return "Update failed its health check and was rolled back to the previous version. It had migrated the database, so the pre-update backup was restored and every write made since the update began is gone." + } if report.RolledBack { return "Update failed its health check and was rolled back to the previous version." } diff --git a/internal/releaseupdate/transaction.go b/internal/releaseupdate/transaction.go new file mode 100644 index 000000000..3a07cfbfc --- /dev/null +++ b/internal/releaseupdate/transaction.go @@ -0,0 +1,565 @@ +package releaseupdate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "time" + + "golang.org/x/sys/unix" +) + +// A transaction's journal moves through these phases. prepared means nothing +// live has changed yet; committed and rolled-back mean the outcome is decided. +// Any other phase found by a recovering watchdog means binaries or the image +// policy may be half switched, and it rolls back. +const ( + phasePrepared = "prepared" + phaseSwitching = "switching" + phaseObserving = "observing" + phaseRollingBack = "rolling-back" + phaseCommitted = "committed" + phaseRolledBack = "rolled-back" +) + +const ( + // stateStoreUnit is started ahead of the others so the image policy can + // be switched before archied reads it. + stateStoreUnit = "archie-state-store" + agentVersionFile = "archie-agent.version" + // WatchdogBinary is the archied copy frozen into the journal: the binary + // that wrote a journal is the one that interprets it, even after the + // executables in the bin directory have been replaced. + WatchdogBinary = "archied" + journalDir = "transaction" + journalFile = "journal.json" + snapshotFile = "db.pre-update.dump" + lockWait = 60 * time.Second +) + +// Units starts and stops the host's services. Managed is false where no +// service manager runs them; then nothing is restarted. +type Units interface { + Managed(ctx context.Context) bool + Start(ctx context.Context, unit string) error + Stop(ctx context.Context, unit string) error +} + +// Images fetches the agent image a release pins. +type Images interface { + Pull(ctx context.Context, ref string) error +} + +// ImagePolicy is the control plane's agent image setting. +type ImagePolicy interface { + Image(ctx context.Context) (string, error) + // SetImage replaces expected with desired. It succeeds without writing + // when desired is already set, and refuses any other current value. + SetImage(ctx context.Context, expected, desired string) error +} + +// Database is the task database the services share. +type Database interface { + Backup(ctx context.Context, out string) error + SchemaVersion(ctx context.Context) (int64, error) + Restore(ctx context.Context, from string) error +} + +// Health asks the daemon whether it is healthy. +type Health interface { + // Healthy returns the version a healthy daemon reports, or why it is not. + Healthy(ctx context.Context) (string, error) +} + +// Transaction installs a release and either commits it or rolls the host back +// to the previous binaries and image policy. Prepare stages everything without +// touching what runs; Run, in a process of its own, switches over, watches the +// daemon and decides. +type Transaction struct { + StateDir string + BinDir string + Units Units + Images Images + Policy ImagePolicy + Database Database + Health Health + Progress func(string) + + // HealthTimeout bounds the wait for the restarted daemon to report the + // release; it must then stay healthy for Observe, checked every Poll. + HealthTimeout time.Duration + Observe time.Duration + Poll time.Duration +} + +type imageChange struct { + Previous string `json:"previous"` + Candidate string `json:"candidate"` +} + +type journal struct { + Phase string `json:"phase"` + Version string `json:"version"` + Previous map[string]string `json:"previous"` + Units []string `json:"units"` + Managed bool `json:"managed"` + Image imageChange `json:"image"` + SchemaBefore int64 `json:"schema_before"` + Meta InstallMeta `json:"meta"` + // RestoredDatabase records that rollback found a migration and restored + // the pre-update snapshot. + RestoredDatabase bool `json:"restored_database"` +} + +// JournalPath is where an unfinished transaction lives. +func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) } + +// Run drives the transaction in the journal to an outcome. recovering is set +// on boot: it never waits for the lock, and a transaction that was prepared +// but never started is abandoned rather than started unattended. +func (t *Transaction) Run(ctx context.Context, recovering bool) error { + unlock, held, err := lockState(ctx, t.StateDir, !recovering) + if err != nil { + return err + } + if !held { + // A live installer or watchdog owns the journal. + return nil + } + defer unlock() + j, found, err := t.load() + if err != nil || !found { + return err + } + switch j.Phase { + case phasePrepared: + if recovering { + return t.abandon(ctx, j) + } + return t.promote(ctx, j) + case phaseCommitted, phaseRolledBack: + // The outcome was decided before the watchdog died. archied exits at + // boot while a journal exists, so the units come up again here. + if err := t.startUnits(ctx, j, ""); err != nil { + return err + } + return t.remove() + default: + return t.rollback(ctx, j) + } +} + +func (t *Transaction) promote(ctx context.Context, j journal) error { + if err := t.setPhase(&j, phaseSwitching); err != nil { + return err + } + if err := t.switchOver(ctx, &j); err != nil { + t.say("update to %s failed: %v; rolling back", j.Version, err) + return t.rollback(ctx, j) + } + if j.Managed { + if err := t.writeReport(j, "passed", false); err != nil { + return err + } + } else { + t.say("installed %s; restart the archie services to run it", j.Version) + } + if err := t.setPhase(&j, phaseCommitted); err != nil { + return err + } + return t.remove() +} + +// switchOver replaces the executables and image policy, then restarts the +// units in release order and requires the daemon to stay healthy. +func (t *Transaction) switchOver(ctx context.Context, j *journal) error { + if err := t.stopUnits(ctx, *j); err != nil { + return err + } + for _, unit := range j.Units { + if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil { + return fmt.Errorf("switch %s: %w", unit, err) + } + } + if err := writeFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n")); err != nil { + return err + } + if err := syncDir(t.BinDir); err != nil { + return err + } + if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil { + return err + } + if !j.Managed { + return nil + } + if err := t.setPhase(j, phaseObserving); err != nil { + return err + } + if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil { + return err + } + return t.observe(ctx, j.Version) +} + +// rollback is safe to repeat from any point: every step converges on the +// previous state whether or not an earlier attempt got that far. +func (t *Transaction) rollback(ctx context.Context, j journal) error { + if err := t.setPhase(&j, phaseRollingBack); err != nil { + return err + } + if err := t.stopUnits(ctx, j); err != nil { + return t.incomplete(err) + } + if err := t.restoreFiles(j); err != nil { + return t.incomplete(err) + } + restored, err := t.restoreDatabase(ctx, j) + if err != nil { + return t.incomplete(err) + } + started := "" + if restored { + // The snapshot predates the policy switch, so it holds the previous image. + j.RestoredDatabase = true + if err := t.setPhase(&j, phaseRollingBack); err != nil { + return t.incomplete(err) + } + t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot()) + } else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil { + return t.incomplete(err) + } else { + started = stateStoreUnit + } + if err := t.writeReport(j, "failed", true); err != nil { + return t.incomplete(err) + } + if err := t.setPhase(&j, phaseRolledBack); err != nil { + return t.incomplete(err) + } + if err := t.startUnits(ctx, j, started); err != nil { + return t.incomplete(err) + } + return t.remove() +} + +// abandon discards a transaction that never switched anything. +func (t *Transaction) abandon(ctx context.Context, j journal) error { + t.say("update to %s was staged but never started; discarding it", j.Version) + for _, unit := range j.Units { + if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + if err := t.writeReport(j, "failed", true); err != nil { + return err + } + if err := t.setPhase(&j, phaseRolledBack); err != nil { + return err + } + if err := t.startUnits(ctx, j, ""); err != nil { + return err + } + return t.remove() +} + +func (t *Transaction) incomplete(err error) error { + return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err) +} + +func (t *Transaction) restoreFiles(j journal) error { + previous := filepath.Join(JournalPath(t.StateDir), "previous") + for _, unit := range j.Units { + live := filepath.Join(t.BinDir, unit) + saved := filepath.Join(previous, unit) + if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) { + if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } else if err := installFile(saved, live, 0o755); err != nil { + return fmt.Errorf("restore %s: %w", unit, err) + } + if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + version := filepath.Join(t.BinDir, agentVersionFile) + saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile) + if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) { + if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } else if err := installFile(saved, version, 0o644); err != nil { + return err + } + return syncDir(t.BinDir) +} + +// restoreDatabase restores the pre-update snapshot only when the candidate +// migrated the schema: the previous binaries refuse a newer schema, and data +// written since the snapshot is otherwise worth keeping. +func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) { + if j.RestoredDatabase { + return true, nil + } + schema, err := t.Database.SchemaVersion(ctx) + if err != nil { + return false, fmt.Errorf("read schema version: %w", err) + } + if schema <= j.SchemaBefore { + return false, nil + } + if err := t.Database.Restore(ctx, t.snapshot()); err != nil { + return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err) + } + return true, nil +} + +// setImage retries while the State Store comes up. +func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error { + if j.Managed { + if err := t.Units.Start(ctx, stateStoreUnit); err != nil { + return err + } + } + deadline := time.Now().Add(t.HealthTimeout) + for { + err := t.Policy.SetImage(ctx, expected, desired) + if err == nil || time.Now().After(deadline) { + return err + } + if err := sleep(ctx, t.Poll); err != nil { + return err + } + } +} + +func (t *Transaction) observe(ctx context.Context, version string) error { + deadline := time.Now().Add(t.HealthTimeout) + for { + reported, err := t.Health.Healthy(ctx) + if err == nil && sameVersion(reported, version) { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err)) + } + if err := sleep(ctx, t.Poll); err != nil { + return err + } + } + t.say("archied is healthy on %s; watching it for %s", version, t.Observe) + until := time.Now().Add(t.Observe) + for time.Now().Before(until) { + if err := sleep(ctx, t.Poll); err != nil { + return err + } + reported, err := t.Health.Healthy(ctx) + if err != nil || !sameVersion(reported, version) { + return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err)) + } + } + return nil +} + +func healthProblem(reported string, err error) error { + if err != nil { + return err + } + return fmt.Errorf("it reports version %q", reported) +} + +func sameVersion(left, right string) bool { + l, ok := comparableVersion(left) + r, _ := comparableVersion(right) + return ok && l == r +} + +// startUnits restarts the units in release order, skipping one already started. +func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error { + if !j.Managed { + return nil + } + for _, unit := range j.Units { + if unit == skip { + continue + } + if err := t.Units.Start(ctx, unit); err != nil { + return fmt.Errorf("start %s: %w", unit, err) + } + } + return nil +} + +// stopUnits stops in reverse release order, so callers go before the services +// they depend on. +func (t *Transaction) stopUnits(ctx context.Context, j journal) error { + if !j.Managed { + return nil + } + for _, unit := range slices.Backward(j.Units) { + if err := t.Units.Stop(ctx, unit); err != nil { + return fmt.Errorf("stop %s: %w", unit, err) + } + } + return nil +} + +func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error { + if j.Meta.ReportPath == "" { + return nil + } + installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version} + return WritePendingReport(j.Meta.ReportPath, Report{ + Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID, + Previous: j.Previous, Installed: installed, + HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase, + }) +} + +func (t *Transaction) staged(unit string) string { + return filepath.Join(t.BinDir, "."+unit+".new") +} + +func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) } + +func (t *Transaction) say(format string, args ...any) { + if t.Progress != nil { + t.Progress(fmt.Sprintf(format, args...)) + } +} + +func (t *Transaction) load() (journal, bool, error) { + data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile)) + if errors.Is(err, os.ErrNotExist) { + return journal{}, false, nil + } + if err != nil { + return journal{}, false, fmt.Errorf("read update journal: %w", err) + } + var j journal + if err := json.Unmarshal(data, &j); err != nil { + return journal{}, false, fmt.Errorf("decode update journal: %w", err) + } + return j, true, nil +} + +func (t *Transaction) setPhase(j *journal, phase string) error { + j.Phase = phase + return writeJournal(JournalPath(t.StateDir), *j) +} + +func writeJournal(dir string, j journal) error { + data, err := json.Marshal(j) + if err != nil { + return err + } + if err := writeFileAtomic(filepath.Join(dir, journalFile), data); err != nil { + return err + } + return syncDir(dir) +} + +// remove retires the journal by rename first, so a crash mid-delete never +// leaves a partial journal behind. +func (t *Transaction) remove() error { + finished := filepath.Join(t.StateDir, "finished") + if err := os.RemoveAll(finished); err != nil { + return err + } + if err := os.Rename(JournalPath(t.StateDir), finished); err != nil { + return err + } + if err := syncDir(t.StateDir); err != nil { + return err + } + return os.RemoveAll(finished) +} + +// lockState takes the update lock. Without wait it reports held=false at once +// when another process owns it. +func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, false, err + } + file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, false, err + } + deadline := time.Now().Add(lockWait) + for { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err == nil { + return func() { _ = file.Close() }, true, nil + } + if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) { + _ = file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, false, nil + } + return nil, false, err + } + if err := sleep(ctx, 100*time.Millisecond); err != nil { + _ = file.Close() + return nil, false, err + } + } +} + +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func syncDir(dir string) error { + file, err := os.Open(dir) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + return file.Sync() +} + +// installFile copies src over dst through a synced temp file and rename, so +// dst is always either the old or the new content. +func installFile(src, dst string, mode os.FileMode) error { + data, err := os.ReadFile(src) + if err != nil { + return err + } + tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp") + if err := writeSynced(tmp, data, mode); err != nil { + return err + } + return os.Rename(tmp, dst) +} + +func writeSynced(path string, data []byte, mode os.FileMode) error { + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return err + } + if _, err := file.Write(data); err != nil { + _ = file.Close() + return err + } + if err := file.Chmod(mode); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + return file.Close() +} diff --git a/internal/releaseupdate/transaction_test.go b/internal/releaseupdate/transaction_test.go new file mode 100644 index 000000000..7429e2920 --- /dev/null +++ b/internal/releaseupdate/transaction_test.go @@ -0,0 +1,288 @@ +package releaseupdate + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +var testUnits = []string{"archie-state-store", "archie-gateway", "archied"} + +const ( + oldImage = "ghcr.io/example/archie-agent@sha256:1111111111111111111111111111111111111111111111111111111111111111" + newImage = "ghcr.io/example/archie-agent@sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +// fakeHost stands in for systemd, Docker, the State Store and the daemon. A +// started State Store on a migrating release bumps the schema, as a real +// candidate's migrations would. +type fakeHost struct { + bin string + managed bool + migrates bool + unhealthy bool + image string + backedUp string + schema int64 + pulled []string + started []string + restored bool +} + +func (h *fakeHost) Managed(context.Context) bool { return h.managed } + +func (h *fakeHost) Start(_ context.Context, unit string) error { + h.started = append(h.started, unit) + if data, _ := os.ReadFile(filepath.Join(h.bin, unit)); h.migrates && unit == stateStoreUnit && bytes.Contains(data, []byte("2.0.0")) { + h.schema = 2 + } + return nil +} + +func (h *fakeHost) Stop(context.Context, string) error { return nil } + +func (h *fakeHost) Pull(_ context.Context, ref string) error { + h.pulled = append(h.pulled, ref) + return nil +} + +func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil } + +func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error { + if h.image != desired && h.image != expected { + return errors.New("image policy changed") + } + h.image = desired + return nil +} + +func (h *fakeHost) Backup(_ context.Context, out string) error { + h.backedUp = h.image + return os.WriteFile(out, []byte("snapshot"), 0o600) +} + +func (h *fakeHost) SchemaVersion(context.Context) (int64, error) { return h.schema, nil } + +func (h *fakeHost) Restore(_ context.Context, from string) error { + if _, err := os.Stat(from); err != nil { + return err + } + h.restored, h.schema, h.image = true, 1, h.backedUp + return nil +} + +func (h *fakeHost) Healthy(context.Context) (string, error) { + if h.unhealthy { + return "", errors.New("503 Service Unavailable") + } + data, err := os.ReadFile(filepath.Join(h.bin, "archied")) + if err != nil { + return "", err + } + return strings.Fields(string(data))[3], nil +} + +// crash leaves the host as a watchdog killed during phase would have. +func crash(t *testing.T, tx *Transaction, h *fakeHost, phase string) { + t.Helper() + j, found, err := tx.load() + if err != nil || !found { + t.Fatalf("no journal to crash: %v", err) + } + switched := map[string]int{phasePrepared: 0, phaseSwitching: 1, phaseRolledBack: 0} + count, partial := switched[phase] + if !partial { + count = len(j.Units) + h.image = newImage + } + for _, unit := range j.Units[:count] { + if err := os.Rename(tx.staged(unit), filepath.Join(tx.BinDir, unit)); err != nil { + t.Fatal(err) + } + } + if phase == phaseRolledBack { + for _, unit := range j.Units { + if err := os.Remove(tx.staged(unit)); err != nil { + t.Fatal(err) + } + } + } + if phase == phaseRollingBack { + if err := installFile(filepath.Join(JournalPath(tx.StateDir), "previous", j.Units[0]), filepath.Join(tx.BinDir, j.Units[0]), 0o755); err != nil { + t.Fatal(err) + } + } + if err := tx.setPhase(&j, phase); err != nil { + t.Fatal(err) + } +} + +func TestTransaction(t *testing.T) { + type outcome struct { + version, image string + report string // health_check, or "" for no report + restoredDatabase bool + } + old := outcome{version: "1.0.0", image: oldImage, report: "failed"} + tests := []struct { + name string + host fakeHost + candidate string // version the release's archied reports + corrupt bool // SHA256SUMS does not match the zip + crashAt string // phase the watchdog dies in, then boot recovers + prepare string // Prepare refusal + want outcome + }{ + {name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}}, + {name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old}, + {name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}}, + {name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}}, + {name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}}, + {name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}}, + {name: "crash before the watchdog started is abandoned", host: fakeHost{managed: true}, crashAt: phasePrepared, want: old}, + {name: "crash while switching rolls back", host: fakeHost{managed: true}, crashAt: phaseSwitching, want: old}, + {name: "crash while observing rolls back", host: fakeHost{managed: true}, crashAt: phaseObserving, want: old}, + {name: "crash while rolling back finishes it", host: fakeHost{managed: true}, crashAt: phaseRollingBack, want: old}, + {name: "crash after commit keeps the release", host: fakeHost{managed: true}, crashAt: phaseCommitted, want: outcome{version: "2.0.0", image: newImage}}, + {name: "crash after rollback keeps the old release", host: fakeHost{managed: true}, crashAt: phaseRolledBack, want: outcome{version: "1.0.0", image: oldImage}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + host := tt.host + host.bin, host.image, host.schema = filepath.Join(root, "bin"), oldImage, 1 + candidate := tt.candidate + if candidate == "" { + candidate = "2.0.0" + } + server := httptest.NewServer(releaseHandler(t, candidate, tt.corrupt)) + defer server.Close() + for _, unit := range testUnits { + writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0")) + } + self := filepath.Join(root, "self") + writeTestFile(t, self, "frozen watchdog") + report := filepath.Join(root, "report.json") + tx := &Transaction{ + StateDir: filepath.Join(root, "state"), BinDir: host.bin, + Units: &host, Images: &host, Policy: &host, Database: &host, Health: &host, + HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond, + } + + _, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}}) + switch { + case tt.prepare != "": + if err == nil || !strings.Contains(err.Error(), tt.prepare) { + t.Fatalf("Prepare error = %v, want %q", err, tt.prepare) + } + case err != nil: + t.Fatalf("Prepare: %v", err) + case tt.crashAt != "": + crash(t, tx, &host, tt.crashAt) + err = tx.Run(t.Context(), true) + default: + err = tx.Run(t.Context(), false) + } + if tt.prepare == "" && err != nil { + t.Fatalf("Run: %v", err) + } + + for _, unit := range testUnits { + data, _ := os.ReadFile(filepath.Join(host.bin, unit)) + if !strings.Contains(string(data), " "+tt.want.version) { + t.Errorf("%s = %q, want %s", unit, data, tt.want.version) + } + if _, err := os.Stat(tx.staged(unit)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("%s is still staged", unit) + } + } + if host.image != tt.want.image { + t.Errorf("image = %s, want %s", host.image, tt.want.image) + } + if _, err := os.Stat(JournalPath(tx.StateDir)); !errors.Is(err, os.ErrNotExist) { + t.Error("journal left behind") + } + if host.restored != tt.want.restoredDatabase { + t.Errorf("database restored = %v, want %v", host.restored, tt.want.restoredDatabase) + } + if host.managed && tt.prepare == "" && !slices.Contains(host.started, "archied") { + t.Errorf("archied never started: %v", host.started) + } + if !host.managed && len(host.started) > 0 { + t.Errorf("started %v without a service manager", host.started) + } + got, found, err := ReadPendingReport(report) + if err != nil { + t.Fatal(err) + } + if wantReport := tt.want.report != "" && tt.crashAt != phaseCommitted && tt.crashAt != phaseRolledBack; found != wantReport || + found && (got.HealthCheck != tt.want.report || got.RolledBack != (tt.want.report == "failed") || got.RestoredDatabase != tt.want.restoredDatabase) { + t.Errorf("report = %+v (found %v), want %+v", got, found, tt.want) + } + }) + } +} + +// releaseHandler serves one release's zip and SHA256SUMS the way GitHub +// release downloads are laid out. +func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler { + t.Helper() + name := "archie-core-archied-v2.0.0-linux-amd64" + var archive bytes.Buffer + writer := zip.NewWriter(&archive) + add := func(path, body string) { + header := &zip.FileHeader{Name: name + "/" + path, Method: zip.Deflate} + header.SetMode(0o755) + file, err := writer.CreateHeader(header) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + for _, unit := range testUnits { + version := "2.0.0" + if unit == "archied" { + version = candidate + } + add(unit, versionScript(unit, version)) + } + add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]},"agent_image":%q}`, strings.Join(testUnits, `","`), newImage)) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(archive.Bytes()) + if corrupt { + sum[0] ^= 0xff + } + mux := http.NewServeMux() + mux.HandleFunc("/v2.0.0/"+name+".zip", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(archive.Bytes()) }) + mux.HandleFunc("/v2.0.0/SHA256SUMS", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, "%x %s.zip\n", sum, name) }) + return mux +} + +func versionScript(unit, version string) string { + return fmt.Sprintf("#!/bin/sh\n# %s %s\necho '%s %s'\n", unit, version, unit, version) +} + +func writeTestFile(t *testing.T, path, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o755); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/archie-update-install b/scripts/archie-update-install deleted file mode 100755 index 7c341df16..000000000 --- a/scripts/archie-update-install +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env bash -# archie-update-install -- component-aware update installation adapter. -# -# CommandInstaller passes the freshly revalidated, approved snapshot through -# ARCHIE_UPDATE_{DAEMON,AGENT}_{PREVIOUS,VERSION}. A VERSION is non-empty only -# when that component changed in the approved snapshot. This script must never -# rediscover a broader plan after approval. -set -euo pipefail - -json_escape() { - printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' -} - -SOURCE_URL="${ARCHIE_SOURCE_URL:-https://github.com/samcharles93/archie-core.git}" -BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}" -CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}" -export PATH="/usr/local/go/bin:$PATH" -# The units load this file; a run from a shell needs it too, for PGPASSWORD. -ENV_FILE="$(dirname "$CONFIG_PATH")/env" -if [ -r "$ENV_FILE" ]; then - set -a - # shellcheck disable=SC1090 - . "$ENV_FILE" - set +a -fi - -# State for the managed image, independent of archied's daemon build stamp. -AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}" -DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}" -# The address the daemon dials for its state, so the watchdog can wait for it to -# accept connections before it starts archied. Read with a narrow awk: the -# daemon stays the authority on validation, and this script deliberately -# understands one scalar rather than parsing TOML. Scoped to [services.state] so the Gateway's own target cannot be -# mistaken for it, and exact on the key so target_token is not. -configured_state_store_target() { - awk ' - /^[[:space:]]*\[services\.state\][[:space:]]*$/ { inside = 1; next } - /^[[:space:]]*\[/ { inside = 0 } - inside && /^[[:space:]]*target[[:space:]]*=/ { - value = $0 - sub(/^[[:space:]]*target[[:space:]]*=[[:space:]]*/, "", value) - sub(/[[:space:]]*(#.*)?$/, "", value) - if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value } - else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2) - exit - } - ' "$CONFIG_PATH" 2>/dev/null || true -} - -STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-$(configured_state_store_target)}" - -GATEWAY_SERVICES="archie-state-store archie-gateway archie-ui archie-messaging archied" -GATEWAY_BINARIES="$GATEWAY_SERVICES" - -SOURCE_BUILD="${ARCHIE_UPDATE_SOURCE_BUILD:-0}" -AUTO=0 -for arg in "$@"; do - case "$arg" in - --from-source) SOURCE_BUILD=1 ;; - --auto) AUTO=1 ;; - *) echo "unknown option: $arg" >&2; exit 1 ;; - esac -done - -if [ "$AUTO" = 1 ] && [ -z "${ARCHIE_UPDATE_DAEMON_VERSION:-}${ARCHIE_UPDATE_AGENT_VERSION:-}" ]; then - snapshot="$("$BIN_DIR/archied" update check)" - export ARCHIE_UPDATE_DAEMON_PREVIOUS="$(jq -r '.Components[] | select(.ID == "daemon") | .Installed' <<< "$snapshot")" - export ARCHIE_UPDATE_DAEMON_VERSION="$(jq -r '.Components[] | select(.ID == "daemon") | .Available' <<< "$snapshot")" - export ARCHIE_UPDATE_AGENT_PREVIOUS="$(jq -r '.Components[] | select(.ID == "agent") | .Installed' <<< "$snapshot")" - export ARCHIE_UPDATE_AGENT_VERSION="$(jq -r '.Components[] | select(.ID == "agent") | .Available' <<< "$snapshot")" -fi - -PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}" -PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}" -GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}" -RT_TARGET="${ARCHIE_UPDATE_AGENT_VERSION:-}" -GW="$PREV_GW" -RT="$PREV_RT" -components="" - -if [ -n "$GW_TARGET" ] && [ "$GW_TARGET" != "$PREV_GW" ]; then - GW="$GW_TARGET" - components="daemon" -fi -if [ -n "$RT_TARGET" ] && [ "$RT_TARGET" != "$PREV_RT" ]; then - case "$RT_TARGET" in - dev|unknown) echo "refusing to build a released runtime image with version '$RT_TARGET'" >&2; exit 1 ;; - esac - RT="$RT_TARGET" - components="${components:+$components,}agent" -fi - -verify_installed_versions() { - reference="$1" - problems="" - for cmd in $GATEWAY_BINARIES; do - [ -f "$BIN_DIR/$cmd" ] || continue - reported="$("$BIN_DIR/$cmd" -version 2>/dev/null | awk -v want="$cmd" '$1 == want { print $2; exit }')" - [ -n "$reported" ] || reported="unknown" - if [ "$reported" != "$reference" ]; then - problems="${problems:+$problems }$cmd=$reported" - fi - done - if [ -n "$problems" ]; then - echo "refusing: installed binaries disagree with $reference: $problems" >&2 - echo "each host binary must report the release it was built from; reinstall the outliers from archied/v$reference, or approve an update that rebuilds them" >&2 - return 1 - fi -} - -if [ -z "$components" ]; then - reference="$PREV_GW" - if [ -z "$reference" ]; then - reference="$("$BIN_DIR/archied" -version 2>/dev/null | awk '$1 == "archied" { print $2; exit }')" - fi - if [ -n "$reference" ] && ! verify_installed_versions "$reference"; then - exit 1 - fi - echo '==> no approved component changes; nothing to install' - echo 'ARCHIE_UPDATE_RESULT {"previous":{},"installed":{},"components":[],"restart_requested":false}' - exit 0 -fi - -valid_release_version() { - [[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$ ]] -} - -for target in "$GW_TARGET" "$RT_TARGET"; do - if [ -n "$target" ] && ! valid_release_version "$target"; then - echo "refusing non-release component version '$target'" >&2 - exit 1 - fi -done - -STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}" -mkdir -p "$STATE_DIR" -chmod 700 "$STATE_DIR" -exec 9>"$STATE_DIR/lock" -if ! flock -n 9; then - echo "update locked by $(cat "$STATE_DIR/holder" 2>/dev/null); journal: $STATE_DIR/transaction" >&2 - exit 1 -fi -printf 'installer PID %s\n' "$$" > "$STATE_DIR/holder" -if [ -d "$STATE_DIR/transaction" ]; then - echo "unfinished update: run $STATE_DIR/transaction/watchdog --recover before retrying" >&2 - exit 1 -fi - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT -source_dir="$tmp/source" -release_dir="$tmp/release" -mkdir -p "$release_dir" -release_version="${GW_TARGET:-$RT_TARGET}" -if [ -n "$GW_TARGET" ] && [ -n "$RT_TARGET" ] && [ "$GW_TARGET" != "$RT_TARGET" ]; then - echo "daemon and agent must name the same release" >&2; exit 1 -fi -if [ "$SOURCE_BUILD" = 1 ]; then - tag="v$release_version" - git ls-remote --exit-code --tags "$SOURCE_URL" "refs/tags/$tag" >/dev/null || tag="archied/v$release_version" - git clone --quiet --depth=1 --branch "$tag" "$SOURCE_URL" "$source_dir" - cp "$source_dir/release.json" "$release_dir/" - GATEWAY_BINARIES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")" - cp "$source_dir"/scripts/archie-update-* "$release_dir/" - if [[ ",$components," == *,daemon,* ]]; then - for cmd in $GATEWAY_BINARIES; do - [[ "$cmd" =~ ^archie[a-z0-9-]*$ ]] || { echo "invalid release unit: $cmd" >&2; exit 1; } - (cd "$source_dir" && go build -ldflags "-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" -o "$release_dir/$cmd" "./cmd/$cmd") - done - fi -else - case "$(uname -s)/$(uname -m)" in - Linux/x86_64) platform=linux-amd64 ;; - *) echo "no published artifact for $(uname -s)/$(uname -m)" >&2; exit 1 ;; - esac - name="archie-core-archied-v$release_version-$platform" - base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/v$release_version" - if ! curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"; then - base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/archied/v$release_version" - curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip" - fi - curl -fLSs --max-time 60 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS" - expected="$(awk -v name="$name.zip" '$2 == name {print $1}' "$tmp/SHA256SUMS")" - [[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "missing or ambiguous archive checksum" >&2; exit 1; } - printf '%s %s\n' "$expected" "$tmp/$name.zip" | sha256sum --check --status || { echo "release checksum mismatch" >&2; exit 1; } - # Reject paths outside the single distribution directory before extracting. - while IFS= read -r entry; do - case "$entry" in "$name/"*) ;; *) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac - case "/$entry/" in */../*|*/./*|*\\*) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac - done < <(unzip -Z1 "$tmp/$name.zip") - if unzip -Z -l "$tmp/$name.zip" | awk '$1 ~ /^l/ {found=1} END {exit !found}'; then - echo "release contains symlinks" >&2; exit 1 - fi - unzip -q "$tmp/$name.zip" -d "$tmp/unpacked" - if find "$tmp/unpacked" -type l | grep -q .; then echo "release contains symlinks" >&2; exit 1; fi - cp -a "$tmp/unpacked/$name/." "$release_dir/" -fi - -# Observation and config parsing belong to the daemon, including on old hosts. -observer="$release_dir/archied" -[ -x "$observer" ] || observer="$BIN_DIR/archied" -"$observer" update-topology -config "$CONFIG_PATH" -release "$release_dir/release.json" -GATEWAY_SERVICES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")" -GATEWAY_BINARIES="$GATEWAY_SERVICES" - -if [[ ",$components," == *,agent,* ]]; then - if [ "$SOURCE_BUILD" = 1 ]; then - docker build --build-arg "RUNTIME_VERSION=$RT" --iidfile "$tmp/image-id" "$source_dir" - AGENT_IMAGE="$(cat "$tmp/image-id")" - else - AGENT_IMAGE="$(jq -er '.agent_image' "$release_dir/release.json")" - [[ "$AGENT_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]] || { echo "refusing agent update: release has no digest-pinned image" >&2; exit 1; } - docker pull "$AGENT_IMAGE" - fi -fi -if [ "$AUTO" = 1 ]; then - status="$(curl -fsS --max-time 10 "${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}/health/detailed")" - [ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || { echo "refusing auto-update: daemon has work in flight or cannot report it" >&2; exit 1; } -fi - -built="" -if [[ ",$components," == *,daemon,* ]]; then - for cmd in $GATEWAY_BINARIES; do - reported="$("$release_dir/$cmd" -version | awk -v cmd="$cmd" '$1 == cmd {print $2; exit}')" - [ "$reported" = "$GW" ] || { echo "candidate $cmd reports $reported, expected $GW" >&2; exit 1; } - done - mkdir -p "$(dirname "$DB_SNAPSHOT")" - "$BIN_DIR/archie-state-store" backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT" - built="$GATEWAY_BINARIES archie-update-install archie-update-watchdog" -fi - -# Freeze recovery code and every prior file before publishing the intent. -journal="$STATE_DIR/preparing" -rm -rf "$journal" -mkdir -p "$journal/previous" -cp "$release_dir/archie-update-watchdog" "$journal/watchdog" -cp "$observer" "$journal/helper" -if [[ ",$components," == *,agent,* ]]; then - "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -prepare "$AGENT_IMAGE" -fi -chmod 700 "$journal/watchdog" -for cmd in $built; do - if [ -f "$BIN_DIR/$cmd" ]; then cp -p "$BIN_DIR/$cmd" "$journal/previous/$cmd"; fi - install -m755 "$release_dir/$cmd" "$BIN_DIR/.$cmd.new" -done -if [ -f "$AGENT_VERSION_FILE" ]; then cp -p "$AGENT_VERSION_FILE" "$journal/agent.version"; fi -# No unit/config changes are made: topology migration is explicitly refused. -export ARCHIE_UPDATE_AUTO="$AUTO" -export ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" -export ARCHIE_BIN_DIR="$BIN_DIR" ARCHIE_CONFIG_PATH="$CONFIG_PATH" -export ARCHIE_UPDATE_COMPONENTS="$components" ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" ARCHIE_UPDATE_BINARIES="$built" -export ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT" -export ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT" -export ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT" -export ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET" -# Bash's quoted declarations preserve spaces without interpreting journal values. -while IFS= read -r var; do declare -p "$var"; done < <(compgen -e | grep -E '^ARCHIE_(UPDATE_|BIN_DIR$|CONFIG_PATH$|HEALTH_URL$|AGENT_VERSION_FILE$|DB_SNAPSHOT$)') > "$journal/environment" -printf 'prepared\n' > "$journal/phase" -sync -f "$BIN_DIR" -sync -f "$journal" -mv "$journal" "$STATE_DIR/transaction" -sync -f "$STATE_DIR" -journal="$STATE_DIR/transaction" -# The independent watchdog holds the lock and owns promotion, restart and rollback. -systemd-run --user --unit=archied-update-restart --collect --quiet \ - --setenv=ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" "$journal/watchdog" -case "$components" in - daemon) - previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}" - installed="{\"daemon\":\"$(json_escape "$GW")\"}" - results='[{"id":"daemon","label":"daemon","status":"updated"}]' - ;; - agent) - previous="{\"agent\":\"$(json_escape "$PREV_RT")\"}" - installed="{\"agent\":\"$(json_escape "$RT")\"}" - results='[{"id":"agent","label":"agent","status":"updated"}]' - ;; - daemon,agent) - previous="{\"daemon\":\"$(json_escape "$PREV_GW")\",\"agent\":\"$(json_escape "$PREV_RT")\"}" - installed="{\"daemon\":\"$(json_escape "$GW")\",\"agent\":\"$(json_escape "$RT")\"}" - results='[{"id":"daemon","label":"daemon","status":"updated"},{"id":"agent","label":"agent","status":"updated"}]' - ;; -esac -echo "ARCHIE_UPDATE_RESULT {\"previous\":$previous,\"installed\":$installed,\"components\":$results,\"restart_requested\":true}" diff --git a/scripts/archie-update-watchdog b/scripts/archie-update-watchdog deleted file mode 100755 index cc7f8224d..000000000 --- a/scripts/archie-update-watchdog +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash -# archie-update-watchdog -- restarts archied after an update, verifies health, -# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS. -# -# Runtime-only updates still restart archied: the restart is the existing -# phase-2 handoff that makes the daemon consume and deliver the health report. -# It does not rebuild or reinstall the gateway binary. -set -uo pipefail - -STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}" -journal="$STATE_DIR/transaction" -mkdir -p "$STATE_DIR" -exec 9>"$STATE_DIR/lock" -if [ "${1:-}" = --recover ]; then - flock -n 9 || exit 0 # A live installer/watchdog owns recovery. -else - flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; } -fi -[ -d "$journal" ] || exit 0 -printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder" -source "$journal/environment" -BIN_DIR="$ARCHIE_BIN_DIR" -CONFIG_PATH="$ARCHIE_CONFIG_PATH" -DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT" -AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE" -HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}" -HEALTH_URL="${HEALTH_URL%/}" -REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}" -HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}" -OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}" -POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}" -COMPONENTS="$ARCHIE_UPDATE_COMPONENTS" -UNITS="$ARCHIE_UPDATE_UNITS" -BINARIES="$ARCHIE_UPDATE_BINARIES" -STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}" -STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}" - -phase() { - printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" && - mv "$journal/phase.new" "$journal/phase" && sync -f "$journal" -} - -json_escape() { - printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' -} - -report_versions() { - case "$COMPONENTS" in - daemon) - previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}" - installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}" - ;; - agent) - previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" - installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" - ;; - daemon,agent) - previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}" - installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}" - ;; - *) - echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2 - return 1 - ;; - esac -} - -write_report() { - local health_check="$1" rolled_back="$2" - [ -z "$REPORT_PATH" ] && return 0 - report_versions || return 1 - mkdir -p "$(dirname "$REPORT_PATH")" - tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")" - cat >"$tmp" <&1)" - return 1 - fi - LAST_HEALTH="$report" - [ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] && - [ "$(jq -r '.status' <<< "$report")" = ok ] -} - -wait_healthy() { - local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS)) - while [ "$SECONDS" -lt "$deadline" ]; do - if healthy; then - deadline=$((SECONDS + OBSERVE_SECONDS)) - while [ "$SECONDS" -lt "$deadline" ]; do - sleep "$POLL_INTERVAL_SECONDS" - healthy || { echo "==> unhealthy while observing: $LAST_HEALTH"; return 1; } - done - return 0 - fi - sleep "$POLL_INTERVAL_SECONDS" - done - echo "==> never became healthy; last report: $LAST_HEALTH" - return 1 -} - -manage_unit() { systemctl --user "$1" "$2.service"; } - -# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp -# costs no process and no dependency, and accepting is exactly the signal this -# needs: the failure it guards against is `dial tcp ...: connect: connection -# refused`, which is the port not being bound at all. -port_accepts() { - (exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null -} - -# Wait for the dependency before starting its callers. -wait_for_state_store() { - [ -n "$STATE_STORE_TARGET" ] || { - echo "==> no State Store target known; starting the rest without waiting for it" >&2 - return 0 - } - local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS)) - while [ "$SECONDS" -lt "$deadline" ]; do - if port_accepts "$STATE_STORE_TARGET"; then - echo "==> $STATE_STORE_TARGET accepts connections" - return 0 - fi - sleep 1 - done - echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2 - return 0 -} - -# start_units brings the deployment up in dependency order: the State Store -# before anything that dials it, the Gateway before archied and the UI. The order -# sequences the calls; wait_for_state_store is what makes it mean "ready". -start_units() { - local unit rc=0 - for unit in $UNITS; do - echo "==> restarting $unit.service" - manage_unit restart "$unit" || rc=1 - if [ "$unit" = archie-state-store ]; then - wait_for_state_store - fi - done - return $rc -} - -# stop_units tears down in reverse, so consumers are gone before the services -# they depend on. -stop_units() { - local unit ordered="" rc=0 - for unit in $UNITS; do - ordered="$unit${ordered:+ $ordered}" - done - for unit in $ordered; do - manage_unit stop "$unit" || rc=1 - done - return $rc -} - -rollback_files() { - local cmd - for cmd in $BINARIES; do - if [ -f "$journal/previous/$cmd" ]; then - cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" && - mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1 - else - rm -f "$BIN_DIR/$cmd" || return 1 - fi - rm -f "$BIN_DIR/.$cmd.new" - done - if [[ ",$COMPONENTS," == *,agent,* ]]; then - if [ -f "$journal/agent.version" ]; then - cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" && - mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1 - else - rm -f "$AGENT_VERSION_FILE" || return 1 - fi - fi - sync -f "$BIN_DIR" -} - -finish() { - phase "$1" || return 1 - # A terminal journal is harmless if power fails before removal. - rm -rf "$STATE_DIR/finished" - mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1 - rm -rf "$STATE_DIR/finished" -} - -rollback() { - phase rolling-back || return 1 - stop_units && rollback_files || return 1 - if [ -f "$journal/image.json" ]; then - manage_unit start archie-state-store && wait_for_state_store && - "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1 - fi - phase rolled-back || return 1 - start_units || return 1 - write_report failed true || return 1 - finish rolled-back -} - -promote() { - if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then - local status - status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2 - [ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2 - fi - phase switching || return 1 - stop_units || return 1 - for cmd in $BINARIES; do - mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1 - done - if [[ ",$COMPONENTS," == *,agent,* ]]; then - printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" && - mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1 - fi - sync -f "$BIN_DIR" || return 1 - if [ -f "$journal/image.json" ]; then - manage_unit start archie-state-store && wait_for_state_store && - "$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1 - fi - phase observing && start_units && wait_healthy -} - -# explain_database_recovery states what a daemon rollback does not do. The -# PostgreSQL database is never restored automatically: goose Down drops tables, -# and a restore is destructive, loses every write after the snapshot and needs -# every service stopped, so it is an operator's decision, not this script's. -explain_database_recovery() { - echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2 - echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2 - echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2 -} - -case "$(cat "$journal/phase")" in - committed) start_units && finish committed; exit $? ;; - rolled-back) start_units && finish rolled-back; exit $? ;; -esac -if [ -z "${1:-}" ] && promote; then - write_report passed false && finish committed - exit $? -fi -if [ "$(cat "$journal/phase")" = prepared ]; then - echo "update was not promoted; leaving running work alone" >&2 - for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done - write_report failed false && finish rolled-back || exit 1 - if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi - exit 1 -fi -echo "==> restoring interrupted or unhealthy update" >&2 -if rollback; then - if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi - exit 0 -fi -echo "rollback incomplete; retained journal: $journal" >&2 -exit 1 diff --git a/tools/release.sh b/tools/release.sh index 0386db91f..8d966b471 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -94,8 +94,7 @@ archied_dirs() { component_dirs ./cmd/archie-ui printf '%s\n' "cmd/archied" "Dockerfile.archied" "cmd/archie-ui" "internal/app/archieui" \ "Taskfile.yml" ".github/workflows/deploy.yml" "install.sh" \ - "scripts/archie-update-install" \ - "scripts/archie-update-watchdog" "deployments/INSTRUCTIONS.md" + "deployments/INSTRUCTIONS.md" } # The UI Service ships with the archied release on purpose (archie-ui shares