feat(releaseupdate): transactional self-update through archied (#1633)
archied update install verifies the release zip, checks every binary reports the release, pulls the agent image by digest, backs up the database in-process and stages the binaries before handing off to archied update watchdog in its own process. The watchdog restarts units in release order, requires archied healthy for 120s and otherwise rolls back binaries and image policy, restoring the database only when a migration ran. Boot recovery resumes the frozen watchdog. Deletes scripts/archie-update-install, scripts/archie-update-watchdog, the updater's --from-source and --auto, CommandInstaller, ScriptInstaller and the update-image and update-topology subcommands.
This commit is contained in:
1 parent
23f87c4cea
commit
417d379ede
33 files changed
+1754
-1114
No files matched your search
@@ -372,7 +372,6 @@ jobs:
|
||||
cp "dist/$cmd" "$name/"
|
||||
done
|
||||
cp deployments/INSTRUCTIONS.md CHANGELOG.md "$name/"
|
||||
cp scripts/archie-update-* "$name/"
|
||||
digest='${{ needs.agent-image.outputs.agent_digest }}'
|
||||
test -n "$digest"
|
||||
jq --arg image "ghcr.io/samcharles93/archie-agent@$digest" '. + {agent_image: $image}' release.json > "$name/release.json"
|
||||
|
||||
+2
-2
@@ -285,8 +285,8 @@ tasks:
|
||||
- "{{.BIN_DIR}}/archie-messaging"
|
||||
- "{{.BIN_DIR}}/archie-agent"
|
||||
cmds:
|
||||
# One flag set for every service, matching scripts/archie-update-install
|
||||
# and the release workflow, so each binary reports the same version.
|
||||
# One flag set for every service, matching the release workflow, so each
|
||||
# binary reports the same version.
|
||||
- for: [archied, archie-gateway, archie-state-store, archie-ui, archie-messaging]
|
||||
cmd: go build -ldflags "{{.SERVICE_LDFLAGS}}" -o {{.BIN_DIR}}/{{.ITEM}} ./cmd/{{.ITEM}}
|
||||
- go build -ldflags "-X github.com/samcharles93/archie-core/internal/app/agentworker.version={{.RUNTIME_VERSION}} -X github.com/samcharles93/archie-core/internal/installtype.buildType=binary" -o {{.BIN_DIR}}/archie-agent ./cmd/archie-agent
|
||||
|
||||
@@ -20,12 +20,6 @@ func main() {
|
||||
if len(args) > 0 && args[0] == "update" {
|
||||
os.Exit(archied.RunUpdate(args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "update-image" {
|
||||
os.Exit(archied.RunUpdateImage(args[1:], os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "update-topology" {
|
||||
os.Exit(archied.RunUpdateTopology(args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
if len(args) > 0 && args[0] == "status" {
|
||||
os.Exit(archied.RunStatus(args[1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
|
||||
+29
-25
@@ -90,12 +90,14 @@ an install (or an upgrade across v1.30.0) needs its unit as well -- see
|
||||
|
||||
### Native self-update
|
||||
|
||||
`archie-update-install` downloads the approved release's Linux/amd64 archive
|
||||
and verifies its `SHA256SUMS` entry before executing or installing it. Releases
|
||||
must include `release.json`, all host binaries, and the two updater scripts.
|
||||
Older releases without this contract cannot be installed through the artifact
|
||||
path. `--from-source` (or `ARCHIE_UPDATE_SOURCE_BUILD=1`) explicitly clones and
|
||||
builds the selected release instead.
|
||||
Install on Settings → Status runs `archied update install -version X`, which
|
||||
downloads the release's Linux/amd64 zip and verifies it against `SHA256SUMS`
|
||||
before unpacking it. Every host binary in the zip must report `X`. It then pulls
|
||||
the agent image `release.json` pins by digest, backs up the database to
|
||||
`db.pre-update.dump` in the state directory, stages the new binaries beside the
|
||||
live ones and hands the transaction to `archied update watchdog` in a process of
|
||||
its own (a transient systemd user unit). Nothing that runs changes until the
|
||||
watchdog takes over.
|
||||
|
||||
`archied update check` finds the newest GitHub release with a linux-amd64 zip on
|
||||
the channel chosen under Settings → Status → Updates: stable, next (includes
|
||||
@@ -103,30 +105,32 @@ prereleases) or a pinned version. From a shell it reads
|
||||
`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`, or `-channel` and `-pin`.
|
||||
The notification destination remains separate from the release channel.
|
||||
|
||||
For scheduled, unattended runs, `archie-update-install --auto` discovers the
|
||||
selected release and refuses if the daemon reports work in flight. It checks
|
||||
again immediately before promotion. A scheduler can set
|
||||
`ARCHIE_UPDATE_RELEASE_CHANNEL` and `ARCHIE_UPDATE_PIN`; nothing schedules updates
|
||||
by default. All services stop through their ordinary graceful shutdown path.
|
||||
|
||||
`release.json` supplies the service order and required config sections to both
|
||||
first-install and self-update. A missing unit or config section produces an
|
||||
ordered migration plan and refuses installation. Apply that plan before retrying.
|
||||
Automatic topology edits are intentionally unsupported.
|
||||
|
||||
The watchdog switches staged executables by rename, restarts services, and
|
||||
requires the candidate daemon's existing readiness registry to remain healthy
|
||||
for 120 seconds. Agent images come from the release manifest by digest. The
|
||||
image switch updates the control-plane policy, preserving other settings;
|
||||
rollback restores the prior image policy with conflict detection. Source builds
|
||||
use the built image's immutable image ID.
|
||||
The watchdog stops the units in reverse `release.json` order, switches the
|
||||
staged executables by rename, switches the control-plane image policy (only the
|
||||
image field, with conflict detection), and restarts the units in order through
|
||||
`systemctl --user`. archied must report the new version and stay healthy for
|
||||
120 seconds. Otherwise the watchdog restores the previous binaries and image
|
||||
policy. Data is not rolled back unless the new release migrated the database:
|
||||
then the previous binaries cannot serve it, so the pre-update backup is restored
|
||||
and the outcome report says that every write since the update began is gone.
|
||||
Without a systemd user manager the watchdog switches binaries and image policy
|
||||
but restarts nothing, and the install reports that the services need a manual
|
||||
restart.
|
||||
|
||||
Transactions live in `${XDG_STATE_HOME:-~/.local/state}/archie/update`, outside
|
||||
the binary directory. `phase`, `environment`, `image.json` (when applicable),
|
||||
and `previous/` describe the transaction. A lock reports the owning process.
|
||||
On daemon boot, an interrupted transaction schedules independent recovery before
|
||||
normal startup. To recover manually, run `transaction/watchdog --recover` from
|
||||
that state directory. A failed recovery keeps the journal for another attempt.
|
||||
the binary directory: `transaction/journal.json`, `transaction/previous/`, and
|
||||
`transaction/archied`, the frozen copy of the archied that wrote the journal. On
|
||||
daemon boot, an interrupted transaction is
|
||||
handed to that frozen watchdog with `-recover` before normal startup: a
|
||||
transaction that never started is discarded, one that was switching or observing
|
||||
is rolled back, and a decided one is finished. To recover manually, run
|
||||
`transaction/archied update watchdog -recover -bin-dir ~/.local/bin` from that
|
||||
state directory. A failed recovery keeps the journal for another attempt.
|
||||
|
||||
Database snapshots remain available for manual recovery. Schema migrations and
|
||||
PostgreSQL data are never automatically reversed.
|
||||
The database backup stays in the state directory after a committed update for
|
||||
manual recovery with `archie-state-store restore`.
|
||||
@@ -36,7 +36,7 @@ by `archied`, since the v1.30.0 extraction. `archied` starts no gateway for them
|
||||
and logs nothing when one is absent, so a host that updates across v1.30.0
|
||||
without this unit runs on with dead channels and a daemon that looks perfectly
|
||||
healthy. `install.sh` writes this unit on a fresh install;
|
||||
`scripts/archie-update-install` refuses an
|
||||
`archied update install` refuses an
|
||||
update when a host is missing a unit and names what is missing, which now fires
|
||||
for a host that drifted or predates this release rather than the normal path.
|
||||
|
||||
|
||||
+4
-5
@@ -89,8 +89,8 @@ fi
|
||||
|
||||
mkdir -p "${ARCHIE_CONFIG_DIR}" "${ARCHIE_DATA_DIR}" "${ARCHIE_BIN_DIR}"
|
||||
|
||||
# 2. Binaries. STAGE ends up holding the binaries, release.json and the
|
||||
# updater scripts, whichever way they were obtained.
|
||||
# 2. Binaries. STAGE ends up holding the binaries and release.json, whichever
|
||||
# way they were obtained.
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "${TMP}"' EXIT
|
||||
|
||||
@@ -111,7 +111,7 @@ if [ "${FROM_SOURCE}" = true ]; then
|
||||
for cmd in $(jq -er '.required_topology.units | join(" ")' "${SRC_DIR}/release.json"); do
|
||||
(cd "${SRC_DIR}" && go build -ldflags "${LDFLAGS}" -o "${STAGE}/${cmd}" "./cmd/${cmd}")
|
||||
done
|
||||
cp "${SRC_DIR}/release.json" "${SRC_DIR}"/scripts/archie-update-* "${STAGE}/"
|
||||
cp "${SRC_DIR}/release.json" "${STAGE}/"
|
||||
else
|
||||
if [ -z "${VERSION}" ]; then
|
||||
tag="$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" | sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p')"
|
||||
@@ -137,8 +137,7 @@ TOPOLOGY_UNITS="$(tr -d '\n' <"${STAGE}/release.json" | sed -n 's/.*"units": *\[
|
||||
for cmd in ${TOPOLOGY_UNITS}; do
|
||||
install -m755 "${STAGE}/${cmd}" "${ARCHIE_BIN_DIR}/${cmd}"
|
||||
done
|
||||
install -m755 "${STAGE}"/archie-update-* "${ARCHIE_BIN_DIR}/"
|
||||
echo " Installed ${TOPOLOGY_UNITS} and the updater to ${ARCHIE_BIN_DIR}/"
|
||||
echo " Installed ${TOPOLOGY_UNITS} to ${ARCHIE_BIN_DIR}/"
|
||||
|
||||
# 3. Configuration. archied setup asks every question and writes config.toml
|
||||
# plus any secrets to the env file beside it; this script asks nothing itself.
|
||||
|
||||
@@ -126,7 +126,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
return 0
|
||||
}
|
||||
|
||||
if recovering, err := resumeUpdate(context.Background()); err != nil {
|
||||
if recovering, err := resumeUpdate(context.Background(), args.cfgPath); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
} else if recovering {
|
||||
|
||||
+199
-43
@@ -7,55 +7,200 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
// RunUpdateTopology observes the host using the candidate's topology contract.
|
||||
func RunUpdateTopology(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update-topology", flag.ContinueOnError)
|
||||
// The watchdog's patience: how long the restarted daemon has to report the
|
||||
// release, and how long it must then stay healthy.
|
||||
const (
|
||||
updateHealthTimeout = 60 * time.Second
|
||||
updateObserve = 120 * time.Second
|
||||
updatePoll = 3 * time.Second
|
||||
)
|
||||
|
||||
const updateUsage = `usage:
|
||||
archied update check [-channel stable|next|exact-pin] [-pin version]
|
||||
archied update install -version X [-config FILE] [-report FILE -channel C -chat ID -thread ID]
|
||||
archied update watchdog [-config FILE] [-recover]`
|
||||
|
||||
// RunUpdate serves `archied update`: check prints the release snapshot,
|
||||
// install verifies and stages a release and hands it to the watchdog, and
|
||||
// watchdog switches over, watches the daemon and commits or rolls back.
|
||||
func RunUpdate(args []string, out, stderr io.Writer) int {
|
||||
if len(args) == 0 {
|
||||
fmt.Fprintln(stderr, updateUsage)
|
||||
return 2
|
||||
}
|
||||
switch args[0] {
|
||||
case "check":
|
||||
return runUpdateCheck(args[1:], out, stderr)
|
||||
case "install":
|
||||
return runUpdateInstall(args[1:], out, stderr)
|
||||
case "watchdog":
|
||||
return runUpdateWatchdog(args[1:], stderr)
|
||||
default:
|
||||
fmt.Fprintln(stderr, updateUsage)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
func runUpdateInstall(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update install", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "bootstrap config file")
|
||||
releasePath := flags.String("release", "", "release manifest")
|
||||
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
|
||||
version := flags.String("version", "", "release version to install")
|
||||
releaseURL := flags.String("release-url", releaseupdate.ReleaseURL, "base URL of the release downloads")
|
||||
var meta releaseupdate.InstallMeta
|
||||
flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome")
|
||||
flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on")
|
||||
flags.Int64Var(&meta.ChatID, "chat", 0, "chat the outcome is reported to")
|
||||
flags.IntVar(&meta.ThreadID, "thread", 0, "thread the outcome is reported to")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if err := checkUpdateTopology(*configPath, *releasePath, out); err != nil {
|
||||
ctx := context.Background()
|
||||
tx, err := hostTransaction(*configPath, filepath.Dir(self), func(line string) { fmt.Fprintln(out, line) })
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
|
||||
Version: *version, ReleaseURL: *releaseURL, Self: self, Meta: meta,
|
||||
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
|
||||
return checkUpdateTopology(ctx, *configPath, required, managed, out)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
|
||||
if err := spawnWatchdog(ctx, result.RestartRequested, frozen, *configPath, false); err != nil {
|
||||
// The journal stays: the next archied boot recovers it.
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if !result.RestartRequested {
|
||||
fmt.Fprintf(out, "no systemd user manager: the watchdog installs %s without restarting; restart the archie services to run it\n", *version)
|
||||
}
|
||||
data, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(out, releaseupdate.ResultSentinel+string(data))
|
||||
return 0
|
||||
}
|
||||
|
||||
func runUpdateWatchdog(args []string, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update watchdog", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
|
||||
recovering := flags.Bool("recover", false, "recover an interrupted transaction at boot")
|
||||
binDir := flags.String("bin-dir", "", "directory of the live binaries")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
tx, err := hostTransaction(*configPath, *binDir, func(line string) { fmt.Fprintln(stderr, line) })
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
if err := tx.Run(context.Background(), *recovering); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func checkUpdateTopology(configPath, releasePath string, out io.Writer) error {
|
||||
data, err := os.ReadFile(releasePath)
|
||||
// hostTransaction wires the transaction to this host: systemd user units,
|
||||
// Docker, the daemon's health endpoint, the control-plane image policy and
|
||||
// the State Store's own backup and restore.
|
||||
func hostTransaction(configPath, binDir string, progress func(string)) (*releaseupdate.Transaction, error) {
|
||||
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if binDir == "" {
|
||||
return nil, errors.New("-bin-dir is required")
|
||||
}
|
||||
return &releaseupdate.Transaction{
|
||||
StateDir: updateStateDir(),
|
||||
BinDir: binDir,
|
||||
Units: systemdUnits{},
|
||||
Images: dockerImages{},
|
||||
Policy: imagePolicy{configPath: configPath},
|
||||
Database: stateDatabase{configPath: configPath},
|
||||
Health: daemonHealth{url: doc.Config.Health.URL()},
|
||||
Progress: progress,
|
||||
HealthTimeout: updateHealthTimeout,
|
||||
Observe: updateObserve,
|
||||
Poll: updatePoll,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// spawnWatchdog runs the frozen watchdog outside this process, so restarting
|
||||
// any service, including the one that asked for the update, cannot kill it.
|
||||
// Under systemd it is a transient unit of its own; without one it is a
|
||||
// detached session.
|
||||
func spawnWatchdog(ctx context.Context, managed bool, binary, configPath string, recovering bool) error {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var manifest struct {
|
||||
Required releaseupdate.Topology `json:"required_topology"`
|
||||
args := []string{"update", "watchdog", "-config", configPath, "-bin-dir", filepath.Dir(exe)}
|
||||
if recovering {
|
||||
args = append(args, "-recover")
|
||||
}
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
stateEnv := "ARCHIE_UPDATE_STATE_DIR=" + updateStateDir()
|
||||
if managed {
|
||||
// The units load the env file beside the config for database and
|
||||
// service credentials; the watchdog needs the same ones.
|
||||
envFile := "EnvironmentFile=-" + filepath.Join(filepath.Dir(configPath), "env")
|
||||
run := append([]string{"--user", "--collect", "--quiet", "--unit=archied-update-watchdog", "--setenv=" + stateEnv, "--property=" + envFile, binary}, args...)
|
||||
if output, err := exec.CommandContext(ctx, "systemd-run", run...).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("start the update watchdog: %w: %s", err, output)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logFile, err := os.OpenFile(filepath.Join(updateStateDir(), "watchdog.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(manifest.Required.Units) == 0 {
|
||||
return errors.New("release has no required_topology")
|
||||
defer func() { _ = logFile.Close() }()
|
||||
cmd := exec.Command(binary, args...) //nolint:noctx // the watchdog must outlive this process
|
||||
cmd.Env = append(os.Environ(), stateEnv)
|
||||
cmd.Stdout, cmd.Stderr = logFile, logFile
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("start the update watchdog: %w", err)
|
||||
}
|
||||
for _, unit := range manifest.Required.Units {
|
||||
if !regexp.MustCompile(`^archie[a-z0-9-]*$`).MatchString(unit) {
|
||||
return fmt.Errorf("invalid release unit %q", unit)
|
||||
}
|
||||
}
|
||||
data, err = os.ReadFile(configPath)
|
||||
return cmd.Process.Release()
|
||||
}
|
||||
|
||||
// checkUpdateTopology refuses a host missing a unit or config section the
|
||||
// release requires. Units are observable only under systemd; without it there
|
||||
// is nothing to restart, so only the config is checked.
|
||||
func checkUpdateTopology(ctx context.Context, configPath string, required releaseupdate.Topology, managed bool, out io.Writer) error {
|
||||
data, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -64,25 +209,34 @@ func checkUpdateTopology(configPath, releasePath string, out io.Writer) error {
|
||||
return fmt.Errorf("observe topology: %w", err)
|
||||
}
|
||||
sections := make(map[string]bool)
|
||||
for _, section := range manifest.Required.ConfigSections {
|
||||
for _, section := range required.ConfigSections {
|
||||
sections[section] = hasConfigSection(document, section)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
output, observeErr := exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
|
||||
units := observedUnits(output, observeErr)
|
||||
plan := manifest.Required.MigrationPlan(units, sections)
|
||||
if len(plan) != 0 {
|
||||
fmt.Fprintln(out, "Required migration, in order:")
|
||||
for _, step := range plan {
|
||||
fmt.Fprintln(out, " -", step)
|
||||
units := make(map[string]bool)
|
||||
var observeErr error
|
||||
if managed {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
var output []byte
|
||||
output, observeErr = exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
|
||||
units = observedUnits(output, observeErr)
|
||||
} else {
|
||||
for _, unit := range required.Units {
|
||||
units[unit] = true
|
||||
}
|
||||
if observeErr != nil {
|
||||
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
|
||||
}
|
||||
return errors.New("refusing partial installation; apply the printed migration before retrying")
|
||||
}
|
||||
return nil
|
||||
plan := required.MigrationPlan(units, sections)
|
||||
if len(plan) == 0 {
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintln(out, "Required migration, in order:")
|
||||
for _, step := range plan {
|
||||
fmt.Fprintln(out, " -", step)
|
||||
}
|
||||
if observeErr != nil {
|
||||
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
|
||||
}
|
||||
return errors.New("refusing partial installation; apply the printed migration before retrying")
|
||||
}
|
||||
|
||||
func hasConfigSection(document map[string]any, section string) bool {
|
||||
@@ -108,10 +262,11 @@ func updateStateDir() string {
|
||||
return filepath.Join(dir, "archie", "update")
|
||||
}
|
||||
|
||||
// Recovery runs independently of the daemon unit: it must be able to stop that
|
||||
// unit and replace its executable without systemd killing the recovery process.
|
||||
func resumeUpdate(ctx context.Context) (bool, error) {
|
||||
journal := filepath.Join(updateStateDir(), "transaction")
|
||||
// resumeUpdate hands an interrupted transaction to the frozen watchdog and
|
||||
// reports that this boot must stop: recovery may replace this executable and
|
||||
// restart this unit, so it runs outside it.
|
||||
func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
|
||||
journal := releaseupdate.JournalPath(updateStateDir())
|
||||
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
@@ -123,14 +278,15 @@ func resumeUpdate(ctx context.Context) (bool, error) {
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) {
|
||||
// A live watchdog owns the transaction and is restarting this unit.
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) }()
|
||||
cmd := exec.CommandContext(ctx, "systemd-run", "--user", "--collect", "--quiet", "--unit=archied-update-recovery", "--setenv=ARCHIE_UPDATE_STATE_DIR="+updateStateDir(), filepath.Join(journal, "watchdog"), "--boot-recover")
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return false, fmt.Errorf("schedule update recovery: %w: %s", err, output)
|
||||
_ = unix.Flock(int(lock.Fd()), unix.LOCK_UN)
|
||||
frozen := filepath.Join(journal, releaseupdate.WatchdogBinary)
|
||||
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
|
||||
return false, fmt.Errorf("schedule update recovery: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -11,18 +11,14 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
// RunUpdate serves `archied update check`, printing the release snapshot as
|
||||
// JSON. The channel defaults to the environment the installer passes on.
|
||||
func RunUpdate(args []string, out, stderr io.Writer) int {
|
||||
if len(args) == 0 || args[0] != "check" {
|
||||
fmt.Fprintln(stderr, "usage: archied update check [-channel stable|next|exact-pin] [-pin version]")
|
||||
return 2
|
||||
}
|
||||
// runUpdateCheck serves `archied update check`, printing the release snapshot
|
||||
// as JSON. The channel defaults to the environment.
|
||||
func runUpdateCheck(args []string, out, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update check", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
channel := flags.String("channel", envOr("ARCHIE_UPDATE_RELEASE_CHANNEL", "stable"), "stable, next, or exact-pin")
|
||||
pin := flags.String("pin", os.Getenv("ARCHIE_UPDATE_PIN"), "exact release version")
|
||||
if err := flags.Parse(args[1:]); err != nil {
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 2
|
||||
}
|
||||
catalog := &releaseupdate.GitHubCatalog{Channel: func(context.Context) (string, string, error) { return *channel, *pin, nil }}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/statestore"
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
)
|
||||
|
||||
// systemdUnits drives the archie user units through systemctl --user.
|
||||
type systemdUnits struct{}
|
||||
|
||||
// Managed reports whether a systemd user manager answers.
|
||||
func (systemdUnits) Managed(ctx context.Context) bool {
|
||||
return systemctl(ctx, "show-environment") == nil
|
||||
}
|
||||
|
||||
func (systemdUnits) Start(ctx context.Context, unit string) error {
|
||||
return systemctl(ctx, "restart", unit+".service")
|
||||
}
|
||||
|
||||
func (systemdUnits) Stop(ctx context.Context, unit string) error {
|
||||
return systemctl(ctx, "stop", unit+".service")
|
||||
}
|
||||
|
||||
func systemctl(ctx context.Context, args ...string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, "systemctl", append([]string{"--user"}, args...)...).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("systemctl --user %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dockerImages pulls through the Docker CLI.
|
||||
type dockerImages struct{}
|
||||
|
||||
func (dockerImages) Pull(ctx context.Context, ref string) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
if output, err := exec.CommandContext(ctx, "docker", "pull", ref).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("docker pull %s: %w: %s", ref, err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stateDatabase is the task database, backed up and restored in-process by
|
||||
// the State Store's own recovery code.
|
||||
type stateDatabase struct{ configPath string }
|
||||
|
||||
func (d stateDatabase) Backup(ctx context.Context, out string) error {
|
||||
_, err := statestore.RunRecovery(ctx, statestore.RecoveryOptions{Operation: statestore.RecoveryBackup, Out: out, Config: d.configPath})
|
||||
return err
|
||||
}
|
||||
|
||||
func (d stateDatabase) Restore(ctx context.Context, from string) error {
|
||||
_, err := statestore.RunRecovery(ctx, statestore.RecoveryOptions{Operation: statestore.RecoveryRestore, From: from, Config: d.configPath})
|
||||
return err
|
||||
}
|
||||
|
||||
func (d stateDatabase) SchemaVersion(ctx context.Context) (int64, error) {
|
||||
return statestore.SchemaVersion(ctx, statestore.RecoveryOptions{Config: d.configPath})
|
||||
}
|
||||
|
||||
// daemonHealth asks archied's /health/detailed, which answers 200 only once
|
||||
// the daemon serves and every readiness probe passes.
|
||||
type daemonHealth struct{ url string }
|
||||
|
||||
func (h daemonHealth) Healthy(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.url+"/health/detailed", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var report struct {
|
||||
Status health.Status `json:"status"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &report) != nil || report.Status != health.StatusOK {
|
||||
return report.Version, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(body)))
|
||||
}
|
||||
return report.Version, nil
|
||||
}
|
||||
@@ -5,14 +5,8 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
@@ -21,34 +15,43 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
type imageChange struct {
|
||||
Previous string `json:"previous"`
|
||||
Candidate string `json:"candidate"`
|
||||
// imagePolicy is the control plane's agent image, reached through the State
|
||||
// Store contract. Only the image field changes, preserving concurrent edits
|
||||
// to unrelated settings.
|
||||
type imagePolicy struct{ configPath string }
|
||||
|
||||
func (p imagePolicy) Image(ctx context.Context) (string, error) {
|
||||
var image string
|
||||
err := p.with(ctx, func(_ pb.ControlPlaneServiceClient, _ *pb.QueryResponse, _ map[string]json.RawMessage, current string) error {
|
||||
image = current
|
||||
return nil
|
||||
})
|
||||
return image, err
|
||||
}
|
||||
|
||||
// RunUpdateImage journals and switches the control-plane image policy. It only
|
||||
// changes the image field, preserving concurrent edits to unrelated settings.
|
||||
func RunUpdateImage(args []string, stderr io.Writer) int {
|
||||
flags := flag.NewFlagSet("update-image", flag.ContinueOnError)
|
||||
flags.SetOutput(stderr)
|
||||
configPath := flags.String("config", "", "bootstrap config")
|
||||
journal := flags.String("journal", "", "transaction directory")
|
||||
candidate := flags.String("prepare", "", "candidate image digest")
|
||||
rollback := flags.Bool("rollback", false, "restore the prior image")
|
||||
if err := flags.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := updateImage(ctx, *configPath, *journal, *candidate, *rollback); err != nil {
|
||||
fmt.Fprintln(stderr, err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
func (p imagePolicy) SetImage(ctx context.Context, expected, desired string) error {
|
||||
return p.with(ctx, func(rpc pb.ControlPlaneServiceClient, response *pb.QueryResponse, policy map[string]json.RawMessage, current string) error {
|
||||
if current == desired {
|
||||
return nil
|
||||
}
|
||||
if current != expected {
|
||||
return errors.New("image policy changed during update; refusing to overwrite it")
|
||||
}
|
||||
var err error
|
||||
if policy["image"], err = json.Marshal(desired); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func updateImage(ctx context.Context, configPath, journal, candidate string, rollback bool) error {
|
||||
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
|
||||
func (p imagePolicy) with(ctx context.Context, use func(pb.ControlPlaneServiceClient, *pb.QueryResponse, map[string]json.RawMessage, string) error) error {
|
||||
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), p.configPath, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -70,48 +73,5 @@ func updateImage(ctx context.Context, configPath, journal, candidate string, rol
|
||||
if err := json.Unmarshal(policy["image"], ¤t); err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(journal, "image.json")
|
||||
if candidate != "" {
|
||||
return prepareImageChange(path, current, candidate)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var change imageChange
|
||||
if err := json.Unmarshal(data, &change); err != nil {
|
||||
return err
|
||||
}
|
||||
expected, desired := change.Previous, change.Candidate
|
||||
if rollback {
|
||||
expected, desired = desired, expected
|
||||
}
|
||||
if current == desired {
|
||||
return nil
|
||||
}
|
||||
if current != expected {
|
||||
return errors.New("image policy changed during update; refusing to overwrite it")
|
||||
}
|
||||
policy["image"], err = json.Marshal(desired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err = json.Marshal(policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = rpc.Command(ctx, &pb.CommandRequest{Kind: controlplane.ContainerRuntimePoliciesKind, Command: "replace", ValueJson: data, ExpectedVersion: response.Resource.Version, Actor: "self-update", Source: "installer", RequestId: fmt.Sprintf("self-update-%x-%d", sha256.Sum256(data), response.Resource.Version)})
|
||||
return err
|
||||
}
|
||||
|
||||
func prepareImageChange(path, current, candidate string) error {
|
||||
if !regexp.MustCompile(`^([a-zA-Z0-9./:_-]+@)?sha256:[a-f0-9]{64}$`).MatchString(candidate) {
|
||||
return errors.New("candidate image must be pinned by digest")
|
||||
}
|
||||
data, err := json.Marshal(imageChange{Previous: current, Candidate: candidate})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o600)
|
||||
return use(rpc, response, policy, current)
|
||||
}
|
||||
@@ -8,18 +8,19 @@ import (
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
)
|
||||
|
||||
func TestUpdateTopologyRefusal(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest := filepath.Join(root, "release.json")
|
||||
config := filepath.Join(root, "config.toml")
|
||||
writeUpdateFixture(t, manifest, `{"required_topology":{"units":["archie-state-store","archie-gateway","archied"],"config_sections":["services.state","services.gateway"]}}`)
|
||||
required := releaseupdate.Topology{Units: []string{"archie-state-store", "archie-gateway", "archied"}, ConfigSections: []string{"services.state", "services.gateway"}}
|
||||
writeUpdateFixture(t, config, "[services.state]\ntarget='localhost:9090'\n")
|
||||
writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\nprintf 'archied.service enabled enabled\\n'\n")
|
||||
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
|
||||
var output bytes.Buffer
|
||||
if err := checkUpdateTopology(config, manifest, &output); err == nil {
|
||||
if err := checkUpdateTopology(t.Context(), config, required, true, &output); err == nil {
|
||||
t.Fatal("partial topology accepted")
|
||||
}
|
||||
for _, missing := range []string{"configure [services.gateway]", "install and enable archie-state-store.service", "install and enable archie-gateway.service"} {
|
||||
@@ -39,6 +40,7 @@ func TestResumeUpdate(t *testing.T) {
|
||||
}
|
||||
called := filepath.Join(root, "called")
|
||||
writeUpdateFixture(t, filepath.Join(root, "systemd-run"), "#!/bin/sh\nprintf '%s\\n' \"$@\" > '"+called+"'\n")
|
||||
writeUpdateFixture(t, filepath.Join(root, "systemctl"), "#!/bin/sh\n")
|
||||
t.Setenv("PATH", root+":"+os.Getenv("PATH"))
|
||||
lock, err := os.OpenFile(filepath.Join(root, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
@@ -50,7 +52,7 @@ func TestResumeUpdate(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resumed, err := resumeUpdate(t.Context())
|
||||
resumed, err := resumeUpdate(t.Context(), filepath.Join(root, "config.toml"))
|
||||
if err != nil || resumed == active {
|
||||
t.Fatalf("resumed=%v err=%v", resumed, err)
|
||||
}
|
||||
@@ -59,7 +61,7 @@ func TestResumeUpdate(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "--boot-recover") {
|
||||
if !strings.Contains(string(data), "-recover") {
|
||||
t.Fatalf("not a rollback: %s", data)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ func componentInstallTypeEnricher(nats config.NATSConfig) func(string) (string,
|
||||
}
|
||||
|
||||
func makeUpdateService(reader controlplanerpc.ResourceReader, cfg config.Config) *releaseupdate.Service {
|
||||
updates := controlplanerpc.UpdateService(reader, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"), cfg.Health.URL())
|
||||
updates := controlplanerpc.UpdateService(reader, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"))
|
||||
updates.Enrich = componentInstallTypeEnricher(cfg.NATS)
|
||||
return updates
|
||||
}
|
||||
|
||||
@@ -17,13 +17,10 @@ type ResolvedConfig struct {
|
||||
Options Options
|
||||
TelegramToken string
|
||||
Telegram config.TelegramConfig
|
||||
// WorkDir, BotUser and HealthURL are not channel transport settings. They
|
||||
// locate this identity's release-announcement and update-report state
|
||||
// files, and give the update installer the daemon health endpoint to poll
|
||||
// after it applies one.
|
||||
// WorkDir and BotUser are not channel transport settings. They locate
|
||||
// this identity's release-announcement and update-report state files.
|
||||
WorkDir string
|
||||
BotUser string
|
||||
HealthURL string
|
||||
ShowToolCalls bool
|
||||
}
|
||||
|
||||
@@ -34,7 +31,6 @@ type projection struct {
|
||||
telegram config.TelegramConfig
|
||||
workDir string
|
||||
botUser string
|
||||
healthURL string
|
||||
showToolCalls bool
|
||||
}
|
||||
|
||||
@@ -65,7 +61,6 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
|
||||
Telegram: proj.telegram,
|
||||
WorkDir: proj.workDir,
|
||||
BotUser: proj.botUser,
|
||||
HealthURL: proj.healthURL,
|
||||
ShowToolCalls: proj.showToolCalls,
|
||||
}, nil
|
||||
}
|
||||
@@ -92,7 +87,6 @@ func project(cfg config.Config) projection {
|
||||
telegram: cfg.Chat.Telegram,
|
||||
workDir: cfg.WorkDir,
|
||||
botUser: cfg.BotUser,
|
||||
healthURL: cfg.Health.URL(),
|
||||
showToolCalls: cfg.Chat.ShowToolCalls,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ func Run(ctx context.Context, o Options) error {
|
||||
settings = messaging.NewSettingsCommand(controlplanerpc.NewSettingsClient(stateStore.ControlPlane())).WithIdentities(stateStore)
|
||||
channelStatusStore, presenceStore = stateStore, stateStore
|
||||
settingsSource, applyReporter, appliedVersion = controlPlaneClient, reporter, channelVersion
|
||||
updates = controlplanerpc.UpdateService(controlPlaneClient, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"), cfg.HealthURL)
|
||||
updates = controlplanerpc.UpdateService(controlPlaneClient, filepath.Join(cfg.WorkDir, "telegram-update-deferrals.json"))
|
||||
}
|
||||
|
||||
if cfg.Options.StateStore.Target == "" {
|
||||
|
||||
@@ -77,7 +77,7 @@ func Run(ctx context.Context, options Options) error {
|
||||
Log: log,
|
||||
Store: tasks,
|
||||
Chat: chat,
|
||||
Updates: controlplanerpc.UpdateService(controlplanerpc.NewRPCClient(tasks.ControlPlane()), filepath.Join(configuration.DefaultWorkDir(), "ui-update-deferrals.json"), ""),
|
||||
Updates: controlplanerpc.UpdateService(controlplanerpc.NewRPCClient(tasks.ControlPlane()), filepath.Join(configuration.DefaultWorkDir(), "ui-update-deferrals.json")),
|
||||
Health: newReadinessRegistry(opts, tasks, chat, chain, provider),
|
||||
ControlPlane: tasks.ControlPlane(),
|
||||
Identities: tasks,
|
||||
|
||||
@@ -107,3 +107,22 @@ func rollbackPostgres(ctx context.Context, url string, options RecoveryOptions)
|
||||
defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }()
|
||||
return replayRevision(ctx, postgres.NewResources(pool), options)
|
||||
}
|
||||
|
||||
// SchemaVersion reads the database's applied migration without migrating or
|
||||
// refusing a schema newer than this binary, so an older binary can tell that
|
||||
// a newer one migrated it.
|
||||
func SchemaVersion(ctx context.Context, options RecoveryOptions) (int64, error) {
|
||||
cfg, err := bootConfig(ctx, options)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if cfg.DatabaseURL == "" {
|
||||
return 0, errors.New("database_url is required: the configuration names no PostgreSQL database")
|
||||
}
|
||||
pool, err := postgres.Open(ctx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer pool.Close()
|
||||
return postgres.SchemaVersion(ctx, pool)
|
||||
}
|
||||
@@ -271,7 +271,11 @@ func formatPendingReport(report releaseupdate.Report, running map[string]string)
|
||||
previousVersions := strings.Join(versions, ", ")
|
||||
|
||||
if report.RolledBack {
|
||||
return fmt.Sprintf("Update failed health check after restarting and was automatically rolled back to the previous version (%s). archie-core is running normally on the previous release.", previousVersions)
|
||||
text := fmt.Sprintf("Update failed health check after restarting and was automatically rolled back to the previous version (%s). archie-core is running normally on the previous release.", previousVersions)
|
||||
if report.RestoredDatabase {
|
||||
text += " The update had migrated the database, so the pre-update backup was restored: every write made since the update began is gone."
|
||||
}
|
||||
return text
|
||||
}
|
||||
return fmt.Sprintf("Update failed health check after restarting and could NOT be rolled back automatically (no backup was found). archie-core may still be running the broken version, or may not be running at all — manual intervention is required. Previous version was %s.", previousVersions)
|
||||
}
|
||||
|
||||
@@ -916,7 +916,7 @@ type Health struct {
|
||||
}
|
||||
|
||||
// URL renders Listen as an address a local caller can dial, which is what
|
||||
// the daemon hands its update tooling (see releaseupdate.CommandInstaller).
|
||||
// the update watchdog polls after restarting the daemon.
|
||||
// A wildcard bind is a valid thing to listen on but not to dial, so it
|
||||
// resolves to localhost, the same substitution the dashboard link makes.
|
||||
func (h Health) URL() string {
|
||||
|
||||
@@ -41,8 +41,7 @@ const (
|
||||
defaultCaptureRatePerSecond = 1.0
|
||||
defaultCaptureRateBurst = 5
|
||||
|
||||
// defaultHealthListen is the daemon's own liveness address. The update
|
||||
// watchdog's fallback in scripts/archie-update-watchdog must match it.
|
||||
// defaultHealthListen is the daemon's own liveness address.
|
||||
defaultHealthListen = "127.0.0.1:8485"
|
||||
|
||||
// defaultHealthDependencyTimeout matches the extracted services'
|
||||
|
||||
@@ -118,12 +118,12 @@ type UpdateSettings struct {
|
||||
}
|
||||
|
||||
// UpdateService checks GitHub on the stored release channel and installs
|
||||
// through the script beside this binary. statePath keeps each recipient's
|
||||
// deferrals; healthURL reaches the installer.
|
||||
func UpdateService(reader ResourceReader, statePath, healthURL string) *releaseupdate.Service {
|
||||
// through the archied beside this binary. statePath keeps each recipient's
|
||||
// deferrals.
|
||||
func UpdateService(reader ResourceReader, statePath string) *releaseupdate.Service {
|
||||
return &releaseupdate.Service{
|
||||
Catalog: &releaseupdate.GitHubCatalog{Channel: updateChannel(reader)},
|
||||
Installer: releaseupdate.ScriptInstaller{HealthURL: healthURL},
|
||||
Installer: releaseupdate.ArchiedInstaller{},
|
||||
StatePath: statePath,
|
||||
InstallType: installtype.Type(),
|
||||
}
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// updateResultSentinel prefixes the one line an install command may print to
|
||||
// report its structured Result. Every other line is forwarded to progress
|
||||
// as-is.
|
||||
const updateResultSentinel = "ARCHIE_UPDATE_RESULT "
|
||||
|
||||
// CommandInstaller runs the deployment's install command. HealthURL is
|
||||
// passed as ARCHIE_HEALTH_URL when set.
|
||||
type CommandInstaller struct {
|
||||
Command []string
|
||||
HealthURL string
|
||||
}
|
||||
|
||||
// Install runs the command, streaming stdout lines to progress. One line may
|
||||
// be a structured Result.
|
||||
func (i CommandInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) {
|
||||
if len(i.Command) == 0 {
|
||||
return Result{}, fmt.Errorf("update install command is empty")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, i.Command[0], i.Command[1:]...)
|
||||
versions := componentVersions(snapshot)
|
||||
cmd.Env = append(
|
||||
cmd.Environ(),
|
||||
"ARCHIE_UPDATE_CHANNEL="+meta.Channel,
|
||||
"ARCHIE_UPDATE_CHAT_ID="+strconv.FormatInt(meta.ChatID, 10),
|
||||
"ARCHIE_UPDATE_THREAD_ID="+strconv.Itoa(meta.ThreadID),
|
||||
"ARCHIE_UPDATE_REPORT_PATH="+meta.ReportPath,
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS="+versions[ComponentDaemon].Installed,
|
||||
"ARCHIE_UPDATE_DAEMON_VERSION="+versions[ComponentDaemon].Available,
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS="+versions[ComponentAgent].Installed,
|
||||
"ARCHIE_UPDATE_AGENT_VERSION="+versions[ComponentAgent].Available,
|
||||
)
|
||||
if i.HealthURL != "" {
|
||||
cmd.Env = append(cmd.Env, "ARCHIE_HEALTH_URL="+i.HealthURL)
|
||||
}
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("open update installer stdout: %w", err)
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return Result{}, fmt.Errorf("start update installer: %w", err)
|
||||
}
|
||||
|
||||
var result Result
|
||||
var resultErr error
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if rest, ok := strings.CutPrefix(line, updateResultSentinel); ok {
|
||||
if err := json.Unmarshal([]byte(rest), &result); err != nil {
|
||||
resultErr = fmt.Errorf("decode update installer result: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
progress(line)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
resultErr = fmt.Errorf("read update installer output: %w", err)
|
||||
}
|
||||
|
||||
waitErr := cmd.Wait()
|
||||
if waitErr != nil {
|
||||
return result, fmt.Errorf("run update installer: %w: %s", waitErr, stderr.String())
|
||||
}
|
||||
if resultErr != nil {
|
||||
return result, resultErr
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// componentVersions reduces an approved snapshot to the two components the
|
||||
// reference binary installer owns. Available is intentionally blank for an
|
||||
// unchanged component: the install script treats these values as the complete
|
||||
// approved plan, not as hints to rediscover work after approval.
|
||||
func componentVersions(snapshot Snapshot) map[string]Component {
|
||||
versions := map[string]Component{
|
||||
ComponentDaemon: {},
|
||||
ComponentAgent: {},
|
||||
}
|
||||
for _, component := range snapshot.Components {
|
||||
if component.ID != ComponentDaemon && component.ID != ComponentAgent {
|
||||
continue
|
||||
}
|
||||
if component.Available == component.Installed {
|
||||
component.Available = ""
|
||||
}
|
||||
versions[component.ID] = component
|
||||
}
|
||||
return versions
|
||||
}
|
||||
|
||||
// ErrNotConfigured is returned when the step's command is unset.
|
||||
var ErrNotConfigured = errors.New("updates are not configured")
|
||||
|
||||
// ScriptInstaller runs the install script a release places beside this binary.
|
||||
type ScriptInstaller struct {
|
||||
HealthURL string
|
||||
}
|
||||
|
||||
func (s ScriptInstaller) path() string {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
path := filepath.Join(filepath.Dir(exe), "archie-update-install")
|
||||
if info, err := os.Stat(path); err != nil || info.Mode()&0o111 == 0 {
|
||||
return ""
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func (s ScriptInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) {
|
||||
path := s.path()
|
||||
if path == "" {
|
||||
return Result{}, ErrNotConfigured
|
||||
}
|
||||
return CommandInstaller{Command: []string{path}, HealthURL: s.HealthURL}.Install(ctx, snapshot, meta, progress)
|
||||
}
|
||||
|
||||
// Installable reports whether the install script is present.
|
||||
func (s ScriptInstaller) Installable(context.Context) bool { return s.path() != "" }
|
||||
@@ -64,7 +64,7 @@ func (c *GitHubCatalog) Check(ctx context.Context) (Snapshot, error) {
|
||||
}
|
||||
return Snapshot{Components: []Component{
|
||||
component("daemon", "Gateway", installedVersion(buildinfo.Version)),
|
||||
component("agent", "Runtime", installedAgent()),
|
||||
component("agent", "Runtime", installedAgent(executableDir())),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
@@ -197,13 +197,9 @@ func installedVersion(version string) string {
|
||||
// installedAgent is the image the installer last switched to, recorded in a
|
||||
// sidecar beside the binaries; without one it is the runtime this release was
|
||||
// cut with. A malformed sidecar is unknown, not guessed.
|
||||
func installedAgent() string {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return installedVersion(buildinfo.Runtime)
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(filepath.Dir(exe), "archie-agent.version"))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
func installedAgent(binDir string) string {
|
||||
data, err := os.ReadFile(filepath.Join(binDir, agentVersionFile))
|
||||
if binDir == "" || errors.Is(err, os.ErrNotExist) {
|
||||
return installedVersion(buildinfo.Runtime)
|
||||
}
|
||||
version := strings.TrimSpace(string(data))
|
||||
@@ -212,3 +208,11 @@ func installedAgent() string {
|
||||
}
|
||||
return installedVersion(version)
|
||||
}
|
||||
|
||||
func executableDir() string {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return filepath.Dir(exe)
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ResultSentinel prefixes the one line `archied update install` prints to
|
||||
// report its Result. Every other line is progress.
|
||||
const ResultSentinel = "ARCHIE_UPDATE_RESULT "
|
||||
|
||||
// ErrNotConfigured is returned when no archied sits beside this binary to
|
||||
// install with.
|
||||
var ErrNotConfigured = errors.New("updates are not configured")
|
||||
|
||||
// ArchiedInstaller runs `archied update install` from beside this binary, so
|
||||
// every service installs through the daemon's one transaction.
|
||||
type ArchiedInstaller struct{}
|
||||
|
||||
func (ArchiedInstaller) path() string {
|
||||
path := filepath.Join(executableDir(), WatchdogBinary)
|
||||
if info, err := os.Stat(path); err != nil || info.Mode()&0o111 == 0 {
|
||||
return ""
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// Installable reports whether archied is present to install with.
|
||||
func (i ArchiedInstaller) Installable(context.Context) bool { return i.path() != "" }
|
||||
|
||||
// Install installs the snapshot's release, streaming progress lines.
|
||||
func (i ArchiedInstaller) Install(ctx context.Context, snapshot Snapshot, meta InstallMeta, progress func(string)) (Result, error) {
|
||||
path := i.path()
|
||||
if path == "" {
|
||||
return Result{}, ErrNotConfigured
|
||||
}
|
||||
available := snapshot.Available()
|
||||
if len(available) == 0 {
|
||||
return Result{}, errors.New("nothing to install")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, path, "update", "install",
|
||||
"-version", available[0].Available,
|
||||
"-report", meta.ReportPath,
|
||||
"-channel", meta.Channel,
|
||||
"-chat", strconv.FormatInt(meta.ChatID, 10),
|
||||
"-thread", strconv.Itoa(meta.ThreadID))
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("open update installer stdout: %w", err)
|
||||
}
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Start(); err != nil {
|
||||
return Result{}, fmt.Errorf("start update installer: %w", err)
|
||||
}
|
||||
var result Result
|
||||
var resultErr error
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if rest, ok := strings.CutPrefix(line, ResultSentinel); ok {
|
||||
if err := json.Unmarshal([]byte(rest), &result); err != nil {
|
||||
resultErr = fmt.Errorf("decode update installer result: %w", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
progress(line)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
resultErr = fmt.Errorf("read update installer output: %w", err)
|
||||
}
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return result, fmt.Errorf("run update installer: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return result, resultErr
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func TestUpdateTransaction(t *testing.T) {
|
||||
for _, scenario := range []string{"healthy", "checksum", "unsafe-path", "missing-digest", "busy", "lock", "unhealthy", "crash"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
f := newUpdateFixture(t, scenario)
|
||||
var lock *os.File
|
||||
if scenario == "lock" {
|
||||
var err error
|
||||
lock, err = os.OpenFile(filepath.Join(f.state, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer lock.Close()
|
||||
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(f.state, "holder"), "fixture-owner", 0o600)
|
||||
}
|
||||
args := []string{}
|
||||
if scenario == "busy" {
|
||||
args = append(args, "--auto")
|
||||
}
|
||||
output, err := f.run(t, f.installer, args...)
|
||||
refusal := map[string]string{"checksum": "checksum mismatch", "unsafe-path": "unsafe archive path", "missing-digest": "digest-pinned", "busy": "work in flight", "lock": "fixture-owner"}[scenario]
|
||||
if refusal != "" {
|
||||
if err == nil || !strings.Contains(output, refusal) {
|
||||
t.Fatalf("wanted %q refusal; err=%v output=%s", refusal, err, output)
|
||||
}
|
||||
f.assertVersion(t, "old")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("installer: %v\n%s", err, output)
|
||||
}
|
||||
f.assertVersion(t, "old") // Staging cannot mutate a running executable.
|
||||
old, err := os.Open(filepath.Join(f.bin, "archied"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer old.Close()
|
||||
watchdog := filepath.Join(f.state, "transaction", "watchdog")
|
||||
output, err = f.run(t, watchdog)
|
||||
if scenario == "crash" {
|
||||
if err == nil {
|
||||
t.Fatal("expected injected crash")
|
||||
}
|
||||
f.env = append(f.env, "CRASH_AFTER=")
|
||||
output, err = f.run(t, watchdog, "--boot-recover")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("watchdog: %v\n%s", err, output)
|
||||
}
|
||||
want := "old"
|
||||
if scenario == "healthy" {
|
||||
want = "new"
|
||||
}
|
||||
f.assertVersion(t, want)
|
||||
data := make([]byte, 100)
|
||||
n, _ := old.Read(data)
|
||||
if !strings.Contains(string(data[:n]), "old") {
|
||||
t.Fatal("promotion overwrote the open old inode")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.state, "transaction")); !os.IsNotExist(err) {
|
||||
t.Fatalf("journal not retired: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type updateFixture struct {
|
||||
bin, state, installer string
|
||||
env []string
|
||||
}
|
||||
|
||||
func fixtureWrite(t *testing.T, path, data string, mode os.FileMode) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(data), mode); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func newUpdateFixture(t *testing.T, scenario string) updateFixture {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
bin, state, fake := filepath.Join(root, "bin"), filepath.Join(root, "state"), filepath.Join(root, "fake")
|
||||
for _, dir := range []string{bin, state, fake} {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
scripts, err := filepath.Abs("../../scripts")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
|
||||
fixtureWrite(t, filepath.Join(bin, name), "#!/bin/bash\n# old\nexit 0\n", 0o755)
|
||||
}
|
||||
config := filepath.Join(root, "config.toml")
|
||||
fixtureWrite(t, config, "[services.state]\n[services.gateway]\n", 0o600)
|
||||
archive := makeUpdateArchive(t, root, scripts, scenario)
|
||||
sum := sha256.Sum256(archive)
|
||||
if scenario == "checksum" {
|
||||
sum[0]++
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(root, "SHA256SUMS"), fmt.Sprintf("%x archie-core-archied-v2.0.0-linux-amd64.zip\n", sum), 0o600)
|
||||
fixtureWrite(t, filepath.Join(fake, "curl"), `#!/bin/bash
|
||||
out=""
|
||||
for ((i=1;i<=$#;i++)); do
|
||||
if [ "${!i}" = -o ]; then j=$((i+1)); out="${!j}"; fi
|
||||
case "${!i}" in
|
||||
*/SHA256SUMS) source="$FIXTURE/SHA256SUMS" ;;
|
||||
*.zip) source="$FIXTURE/archive.zip" ;;
|
||||
esac
|
||||
done
|
||||
if [ -n "$out" ]; then cp "$source" "$out"; exit; fi
|
||||
if [ "$SCENARIO" = unhealthy ]; then exit 22; fi
|
||||
busy=0
|
||||
[ "$SCENARIO" = busy ] && busy=1
|
||||
printf '{"status":"ok","version":"2.0.0","in_flight":%s}\n' "$busy"
|
||||
`, 0o755)
|
||||
for _, name := range []string{"systemctl", "systemd-run", "docker"} {
|
||||
fixtureWrite(t, filepath.Join(fake, name), "#!/bin/bash\nexit 0\n", 0o755)
|
||||
}
|
||||
fixtureWrite(t, filepath.Join(fake, "git"), "#!/bin/bash\necho unexpected-git >&2\nexit 98\n", 0o755)
|
||||
fixtureWrite(t, filepath.Join(fake, "mv"), `#!/bin/bash
|
||||
/bin/mv "$@" || exit
|
||||
if [ -n "$CRASH_AFTER" ] && [[ "$*" == *".archied.new"* ]]; then kill -KILL "$PPID"; fi
|
||||
`, 0o755)
|
||||
return updateFixture{bin: bin, state: state, installer: filepath.Join(scripts, "archie-update-install"), env: append(os.Environ(),
|
||||
"PATH="+fake+":"+os.Getenv("PATH"), "FIXTURE="+root, "SCENARIO="+scenario,
|
||||
"ARCHIE_BIN_DIR="+bin, "ARCHIE_CONFIG_PATH="+config, "ARCHIE_UPDATE_STATE_DIR="+state,
|
||||
"ARCHIE_UPDATE_DAEMON_PREVIOUS=1.0.0", "ARCHIE_UPDATE_DAEMON_VERSION=2.0.0",
|
||||
"ARCHIE_UPDATE_AGENT_PREVIOUS=1.0.0", "ARCHIE_UPDATE_AGENT_VERSION="+map[bool]string{true: "2.0.0"}[scenario == "missing-digest"],
|
||||
"ARCHIE_UPDATE_HEALTH_TIMEOUT=1", "ARCHIE_UPDATE_OBSERVE_SECONDS=0", "ARCHIE_UPDATE_POLL_INTERVAL=0.01",
|
||||
"ARCHIE_UPDATE_STATE_STORE_TARGET=", "CRASH_AFTER="+map[bool]string{true: "archied"}[scenario == "crash"],
|
||||
)}
|
||||
}
|
||||
|
||||
func makeUpdateArchive(t *testing.T, root, scripts, scenario string) []byte {
|
||||
t.Helper()
|
||||
file, err := os.Create(filepath.Join(root, "archive.zip"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := zip.NewWriter(file)
|
||||
files := map[string]string{"release.json": `{"required_topology":{"units":["archie-state-store","archie-gateway","archie-ui","archie-messaging","archied"],"config_sections":["services.state","services.gateway"]},"agent_image":"mutable:latest"}`}
|
||||
for _, name := range []string{"archied", "archie-state-store", "archie-gateway", "archie-ui", "archie-messaging"} {
|
||||
files[name] = "#!/bin/bash\n# new\nif [ \"$1\" = -version ]; then echo '" + name + " 2.0.0'; fi\nexit 0\n"
|
||||
}
|
||||
for _, name := range []string{"archie-update-install", "archie-update-watchdog"} {
|
||||
data, err := os.ReadFile(filepath.Join(scripts, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files[name] = string(data)
|
||||
}
|
||||
if scenario == "unsafe-path" {
|
||||
files["../escape"] = "escape"
|
||||
}
|
||||
for name, body := range files {
|
||||
header := &zip.FileHeader{Name: "archie-core-archied-v2.0.0-linux-amd64/" + name, Method: zip.Deflate}
|
||||
header.SetMode(0o755)
|
||||
w, err := z.CreateHeader(header)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := os.ReadFile(file.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func (f updateFixture) run(t *testing.T, path string, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := exec.CommandContext(t.Context(), path, args...)
|
||||
cmd.Env = f.env
|
||||
data, err := cmd.CombinedOutput()
|
||||
return string(data), err
|
||||
}
|
||||
|
||||
func (f updateFixture) assertVersion(t *testing.T, want string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join(f.bin, "archied"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(data), "# "+want) {
|
||||
t.Fatalf("wanted %s executable: %s", want, data)
|
||||
}
|
||||
}
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// Report is an update's post-restart outcome, written by the restart tooling
|
||||
// and relayed by the next archied boot.
|
||||
// Report is an update's post-restart outcome, written by the watchdog and
|
||||
// relayed by the next archied boot.
|
||||
type Report struct {
|
||||
Channel string `json:"channel"`
|
||||
ChatID int64 `json:"chat_id"`
|
||||
@@ -17,6 +17,9 @@ type Report struct {
|
||||
Installed map[string]string `json:"installed"`
|
||||
HealthCheck string `json:"health_check"` // "passed" or "failed"
|
||||
RolledBack bool `json:"rolled_back"`
|
||||
// RestoredDatabase reports that rollback found a migration and restored
|
||||
// the pre-update snapshot, losing every write made since it.
|
||||
RestoredDatabase bool `json:"restored_database"`
|
||||
}
|
||||
|
||||
func (r Report) validate() error {
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
)
|
||||
|
||||
// ReleaseURL is where published release assets are downloaded from.
|
||||
const ReleaseURL = "https://github.com/" + ReleaseSlug + "/releases/download"
|
||||
|
||||
var (
|
||||
releaseVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$`)
|
||||
releaseUnit = regexp.MustCompile(`^archie[a-z0-9-]*$`)
|
||||
pinnedImage = regexp.MustCompile(`^[a-zA-Z0-9./:_-]+@sha256:[a-f0-9]{64}$`)
|
||||
)
|
||||
|
||||
// PrepareRequest names the release to stage and who hears the outcome.
|
||||
type PrepareRequest struct {
|
||||
Version string
|
||||
// ReleaseURL is the base the release's zip and SHA256SUMS are under.
|
||||
ReleaseURL string
|
||||
// Self is the archied executable that will run the watchdog.
|
||||
Self string
|
||||
Meta InstallMeta
|
||||
// CheckTopology refuses a host missing a unit or config section the
|
||||
// release requires. managed reports whether units are observable.
|
||||
CheckTopology func(ctx context.Context, required Topology, managed bool) error
|
||||
}
|
||||
|
||||
// Manifest is the release.json a release zip carries.
|
||||
type Manifest struct {
|
||||
Required Topology `json:"required_topology"`
|
||||
AgentImage string `json:"agent_image"`
|
||||
}
|
||||
|
||||
// Prepare verifies and stages a release without changing anything that runs:
|
||||
// the zip is checked against SHA256SUMS, every binary must report the release,
|
||||
// the agent image is pulled by digest and the database is backed up before a
|
||||
// journal is published. The watchdog (Run) takes it from there.
|
||||
func (t *Transaction) Prepare(ctx context.Context, req PrepareRequest) (Result, error) {
|
||||
if !releaseVersion.MatchString(req.Version) {
|
||||
return Result{}, fmt.Errorf("refusing non-release version %q", req.Version)
|
||||
}
|
||||
unlock, held, err := lockState(ctx, t.StateDir, false)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if !held {
|
||||
return Result{}, ErrInstallInProgress
|
||||
}
|
||||
defer unlock()
|
||||
if _, err := os.Stat(JournalPath(t.StateDir)); err == nil {
|
||||
return Result{}, fmt.Errorf("an unfinished update is pending recovery at %s", JournalPath(t.StateDir))
|
||||
}
|
||||
work, err := os.MkdirTemp(t.StateDir, ".release-*")
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer func() { _ = os.RemoveAll(work) }()
|
||||
|
||||
release, manifest, err := t.fetch(ctx, req, work)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
managed := t.Units.Managed(ctx)
|
||||
if req.CheckTopology != nil {
|
||||
if err := req.CheckTopology(ctx, manifest.Required, managed); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
j, err := t.snapshotHost(ctx, req, manifest, managed)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if err := t.stage(req, release, j); err != nil {
|
||||
for _, unit := range j.Units {
|
||||
_ = os.Remove(t.staged(unit))
|
||||
}
|
||||
return Result{}, err
|
||||
}
|
||||
installed := map[string]string{ComponentDaemon: req.Version, ComponentAgent: req.Version}
|
||||
return Result{
|
||||
Previous: j.Previous,
|
||||
Installed: installed,
|
||||
Components: []ComponentResult{
|
||||
{ID: ComponentDaemon, Label: "Gateway", Status: "updated"},
|
||||
{ID: ComponentAgent, Label: "Runtime", Status: "updated"},
|
||||
},
|
||||
RestartRequested: managed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fetch downloads, verifies and unpacks the release, and checks every binary
|
||||
// reports the version being installed.
|
||||
func (t *Transaction) fetch(ctx context.Context, req PrepareRequest, work string) (string, Manifest, error) {
|
||||
if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
|
||||
return "", Manifest{}, fmt.Errorf("no published release for %s/%s", runtime.GOOS, runtime.GOARCH)
|
||||
}
|
||||
name := fmt.Sprintf("archie-core-archied-v%s-linux-amd64", req.Version)
|
||||
base := strings.TrimSuffix(req.ReleaseURL, "/") + "/v" + req.Version + "/"
|
||||
t.say("downloading %s", name+".zip")
|
||||
archive := filepath.Join(work, name+".zip")
|
||||
if err := downloadVerified(ctx, base, name+".zip", archive); err != nil {
|
||||
return "", Manifest{}, err
|
||||
}
|
||||
if err := unpack(archive, name, work); err != nil {
|
||||
return "", Manifest{}, err
|
||||
}
|
||||
release := filepath.Join(work, name)
|
||||
manifest, err := readManifest(filepath.Join(release, "release.json"))
|
||||
if err != nil {
|
||||
return "", Manifest{}, err
|
||||
}
|
||||
for _, unit := range manifest.Required.Units {
|
||||
reported, err := reportedVersion(ctx, filepath.Join(release, unit), unit)
|
||||
if err != nil {
|
||||
return "", Manifest{}, err
|
||||
}
|
||||
if !sameVersion(reported, req.Version) {
|
||||
return "", Manifest{}, fmt.Errorf("candidate %s reports %q, not %s", unit, reported, req.Version)
|
||||
}
|
||||
}
|
||||
return release, manifest, nil
|
||||
}
|
||||
|
||||
// snapshotHost pulls the image and records what rollback needs: the current
|
||||
// versions, the current image policy, the schema version and a backup.
|
||||
func (t *Transaction) snapshotHost(ctx context.Context, req PrepareRequest, manifest Manifest, managed bool) (journal, error) {
|
||||
t.say("pulling %s", manifest.AgentImage)
|
||||
if err := t.Images.Pull(ctx, manifest.AgentImage); err != nil {
|
||||
return journal{}, fmt.Errorf("pull agent image: %w", err)
|
||||
}
|
||||
image, err := t.Policy.Image(ctx)
|
||||
if err != nil {
|
||||
return journal{}, fmt.Errorf("read image policy: %w", err)
|
||||
}
|
||||
schema, err := t.Database.SchemaVersion(ctx)
|
||||
if err != nil {
|
||||
return journal{}, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
t.say("backing up the database to %s", t.snapshot())
|
||||
if err := t.Database.Backup(ctx, t.snapshot()); err != nil {
|
||||
return journal{}, fmt.Errorf("back up the database: %w", err)
|
||||
}
|
||||
return journal{
|
||||
Phase: phasePrepared,
|
||||
Version: req.Version,
|
||||
Previous: map[string]string{
|
||||
ComponentDaemon: installedVersion(buildinfo.Version),
|
||||
ComponentAgent: installedAgent(t.BinDir),
|
||||
},
|
||||
Units: manifest.Required.Units,
|
||||
Managed: managed,
|
||||
Image: imageChange{Previous: image, Candidate: manifest.AgentImage},
|
||||
SchemaBefore: schema,
|
||||
Meta: req.Meta,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// stage freezes the watchdog and every file rollback restores, places the
|
||||
// candidates beside the live binaries, and publishes the journal last.
|
||||
func (t *Transaction) stage(req PrepareRequest, release string, j journal) error {
|
||||
preparing := filepath.Join(t.StateDir, "preparing")
|
||||
if err := os.RemoveAll(preparing); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(preparing, "previous"), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := installFile(req.Self, filepath.Join(preparing, WatchdogBinary), 0o700); err != nil {
|
||||
return fmt.Errorf("freeze the watchdog: %w", err)
|
||||
}
|
||||
for _, unit := range j.Units {
|
||||
live := filepath.Join(t.BinDir, unit)
|
||||
if _, err := os.Stat(live); err == nil {
|
||||
if err := installFile(live, filepath.Join(preparing, "previous", unit), 0o755); err != nil {
|
||||
return fmt.Errorf("save %s: %w", unit, err)
|
||||
}
|
||||
}
|
||||
if err := installFile(filepath.Join(release, unit), t.staged(unit), 0o755); err != nil {
|
||||
return fmt.Errorf("stage %s: %w", unit, err)
|
||||
}
|
||||
}
|
||||
version := filepath.Join(t.BinDir, agentVersionFile)
|
||||
if _, err := os.Stat(version); err == nil {
|
||||
if err := installFile(version, filepath.Join(preparing, agentVersionFile), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := syncDir(t.BinDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeJournal(preparing, j); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syncDir(filepath.Join(preparing, "previous")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(preparing, JournalPath(t.StateDir)); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(t.StateDir)
|
||||
}
|
||||
|
||||
func downloadVerified(ctx context.Context, base, name, dst string) error {
|
||||
var sums bytes.Buffer
|
||||
if err := download(ctx, base+"SHA256SUMS", &sums, 1<<20); err != nil {
|
||||
return err
|
||||
}
|
||||
expected := ""
|
||||
scanner := bufio.NewScanner(&sums)
|
||||
for scanner.Scan() {
|
||||
if fields := strings.Fields(scanner.Text()); len(fields) == 2 && fields[1] == name {
|
||||
if expected != "" {
|
||||
return fmt.Errorf("SHA256SUMS lists %s twice", name)
|
||||
}
|
||||
expected = strings.ToLower(fields[0])
|
||||
}
|
||||
}
|
||||
if len(expected) != sha256.Size*2 {
|
||||
return fmt.Errorf("SHA256SUMS has no checksum for %s", name)
|
||||
}
|
||||
file, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hash := sha256.New()
|
||||
err = download(ctx, base+name, io.MultiWriter(file, hash), 1<<30)
|
||||
if closeErr := file.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hex.EncodeToString(hash.Sum(nil)) != expected {
|
||||
return fmt.Errorf("release checksum mismatch for %s", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func download(ctx context.Context, url string, out io.Writer, limit int64) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download %s: %w", url, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download %s: %s", url, resp.Status)
|
||||
}
|
||||
_, err = io.Copy(out, io.LimitReader(resp.Body, limit))
|
||||
return err
|
||||
}
|
||||
|
||||
// unpack extracts the zip's single distribution directory, refusing any
|
||||
// entry outside it and anything but regular files and directories.
|
||||
func unpack(archive, name, dst string) error {
|
||||
reader, err := zip.OpenReader(archive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
for _, file := range reader.File {
|
||||
clean := path.Clean(file.Name)
|
||||
if !strings.HasPrefix(file.Name, name+"/") || strings.Contains(file.Name, `\`) ||
|
||||
clean != strings.TrimSuffix(file.Name, "/") || !strings.HasPrefix(clean, name) {
|
||||
return fmt.Errorf("unsafe archive path %q", file.Name)
|
||||
}
|
||||
mode := file.Mode()
|
||||
target := filepath.Join(dst, filepath.FromSlash(clean))
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
if err := os.MkdirAll(target, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
case mode.IsRegular():
|
||||
if err := extract(file, target); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("archive entry %q is not a regular file", file.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extract(file *zip.File, target string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = src.Close() }()
|
||||
dst, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(dst, io.LimitReader(src, 1<<30)); err != nil {
|
||||
_ = dst.Close()
|
||||
return err
|
||||
}
|
||||
return dst.Close()
|
||||
}
|
||||
|
||||
func readManifest(path string) (Manifest, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
var manifest Manifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
return Manifest{}, fmt.Errorf("decode release.json: %w", err)
|
||||
}
|
||||
if len(manifest.Required.Units) == 0 {
|
||||
return Manifest{}, errors.New("release has no required_topology")
|
||||
}
|
||||
for _, unit := range manifest.Required.Units {
|
||||
if !releaseUnit.MatchString(unit) {
|
||||
return Manifest{}, fmt.Errorf("invalid release unit %q", unit)
|
||||
}
|
||||
}
|
||||
if !pinnedImage.MatchString(manifest.AgentImage) {
|
||||
return Manifest{}, errors.New("release has no digest-pinned agent image")
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// reportedVersion runs binary -version and returns the version on the line
|
||||
// naming unit.
|
||||
func reportedVersion(ctx context.Context, binary, unit string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, binary, "-version").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s -version: %w", unit, err)
|
||||
}
|
||||
for line := range strings.SplitSeq(string(output), "\n") {
|
||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == unit {
|
||||
return fields[1], nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s -version names no version", unit)
|
||||
}
|
||||
@@ -24,13 +24,14 @@ func ReportEvent(report Report, running map[string]string) events.Event {
|
||||
Kind: events.KindUpdateReport,
|
||||
Detail: summarizeReport(report, verification),
|
||||
Data: map[string]any{
|
||||
"health_check": report.HealthCheck,
|
||||
"rolled_back": report.RolledBack,
|
||||
"previous": report.Previous,
|
||||
"installed": report.Installed,
|
||||
"confirmed": verification.Confirmed,
|
||||
"drift": drift,
|
||||
"unverified": verification.Unverified,
|
||||
"health_check": report.HealthCheck,
|
||||
"rolled_back": report.RolledBack,
|
||||
"restored_database": report.RestoredDatabase,
|
||||
"previous": report.Previous,
|
||||
"installed": report.Installed,
|
||||
"confirmed": verification.Confirmed,
|
||||
"drift": drift,
|
||||
"unverified": verification.Unverified,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -38,6 +39,9 @@ func ReportEvent(report Report, running map[string]string) events.Event {
|
||||
// summarizeReport returns a one-line summary: confirmed, drifted, unverified
|
||||
// or rolled back.
|
||||
func summarizeReport(report Report, verification Verification) string {
|
||||
if report.RolledBack && report.RestoredDatabase {
|
||||
return "Update failed its health check and was rolled back to the previous version. It had migrated the database, so the pre-update backup was restored and every write made since the update began is gone."
|
||||
}
|
||||
if report.RolledBack {
|
||||
return "Update failed its health check and was rolled back to the previous version."
|
||||
}
|
||||
|
||||
@@ -0,0 +1,565 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// A transaction's journal moves through these phases. prepared means nothing
|
||||
// live has changed yet; committed and rolled-back mean the outcome is decided.
|
||||
// Any other phase found by a recovering watchdog means binaries or the image
|
||||
// policy may be half switched, and it rolls back.
|
||||
const (
|
||||
phasePrepared = "prepared"
|
||||
phaseSwitching = "switching"
|
||||
phaseObserving = "observing"
|
||||
phaseRollingBack = "rolling-back"
|
||||
phaseCommitted = "committed"
|
||||
phaseRolledBack = "rolled-back"
|
||||
)
|
||||
|
||||
const (
|
||||
// stateStoreUnit is started ahead of the others so the image policy can
|
||||
// be switched before archied reads it.
|
||||
stateStoreUnit = "archie-state-store"
|
||||
agentVersionFile = "archie-agent.version"
|
||||
// WatchdogBinary is the archied copy frozen into the journal: the binary
|
||||
// that wrote a journal is the one that interprets it, even after the
|
||||
// executables in the bin directory have been replaced.
|
||||
WatchdogBinary = "archied"
|
||||
journalDir = "transaction"
|
||||
journalFile = "journal.json"
|
||||
snapshotFile = "db.pre-update.dump"
|
||||
lockWait = 60 * time.Second
|
||||
)
|
||||
|
||||
// Units starts and stops the host's services. Managed is false where no
|
||||
// service manager runs them; then nothing is restarted.
|
||||
type Units interface {
|
||||
Managed(ctx context.Context) bool
|
||||
Start(ctx context.Context, unit string) error
|
||||
Stop(ctx context.Context, unit string) error
|
||||
}
|
||||
|
||||
// Images fetches the agent image a release pins.
|
||||
type Images interface {
|
||||
Pull(ctx context.Context, ref string) error
|
||||
}
|
||||
|
||||
// ImagePolicy is the control plane's agent image setting.
|
||||
type ImagePolicy interface {
|
||||
Image(ctx context.Context) (string, error)
|
||||
// SetImage replaces expected with desired. It succeeds without writing
|
||||
// when desired is already set, and refuses any other current value.
|
||||
SetImage(ctx context.Context, expected, desired string) error
|
||||
}
|
||||
|
||||
// Database is the task database the services share.
|
||||
type Database interface {
|
||||
Backup(ctx context.Context, out string) error
|
||||
SchemaVersion(ctx context.Context) (int64, error)
|
||||
Restore(ctx context.Context, from string) error
|
||||
}
|
||||
|
||||
// Health asks the daemon whether it is healthy.
|
||||
type Health interface {
|
||||
// Healthy returns the version a healthy daemon reports, or why it is not.
|
||||
Healthy(ctx context.Context) (string, error)
|
||||
}
|
||||
|
||||
// Transaction installs a release and either commits it or rolls the host back
|
||||
// to the previous binaries and image policy. Prepare stages everything without
|
||||
// touching what runs; Run, in a process of its own, switches over, watches the
|
||||
// daemon and decides.
|
||||
type Transaction struct {
|
||||
StateDir string
|
||||
BinDir string
|
||||
Units Units
|
||||
Images Images
|
||||
Policy ImagePolicy
|
||||
Database Database
|
||||
Health Health
|
||||
Progress func(string)
|
||||
|
||||
// HealthTimeout bounds the wait for the restarted daemon to report the
|
||||
// release; it must then stay healthy for Observe, checked every Poll.
|
||||
HealthTimeout time.Duration
|
||||
Observe time.Duration
|
||||
Poll time.Duration
|
||||
}
|
||||
|
||||
type imageChange struct {
|
||||
Previous string `json:"previous"`
|
||||
Candidate string `json:"candidate"`
|
||||
}
|
||||
|
||||
type journal struct {
|
||||
Phase string `json:"phase"`
|
||||
Version string `json:"version"`
|
||||
Previous map[string]string `json:"previous"`
|
||||
Units []string `json:"units"`
|
||||
Managed bool `json:"managed"`
|
||||
Image imageChange `json:"image"`
|
||||
SchemaBefore int64 `json:"schema_before"`
|
||||
Meta InstallMeta `json:"meta"`
|
||||
// RestoredDatabase records that rollback found a migration and restored
|
||||
// the pre-update snapshot.
|
||||
RestoredDatabase bool `json:"restored_database"`
|
||||
}
|
||||
|
||||
// JournalPath is where an unfinished transaction lives.
|
||||
func JournalPath(stateDir string) string { return filepath.Join(stateDir, journalDir) }
|
||||
|
||||
// Run drives the transaction in the journal to an outcome. recovering is set
|
||||
// on boot: it never waits for the lock, and a transaction that was prepared
|
||||
// but never started is abandoned rather than started unattended.
|
||||
func (t *Transaction) Run(ctx context.Context, recovering bool) error {
|
||||
unlock, held, err := lockState(ctx, t.StateDir, !recovering)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !held {
|
||||
// A live installer or watchdog owns the journal.
|
||||
return nil
|
||||
}
|
||||
defer unlock()
|
||||
j, found, err := t.load()
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
switch j.Phase {
|
||||
case phasePrepared:
|
||||
if recovering {
|
||||
return t.abandon(ctx, j)
|
||||
}
|
||||
return t.promote(ctx, j)
|
||||
case phaseCommitted, phaseRolledBack:
|
||||
// The outcome was decided before the watchdog died. archied exits at
|
||||
// boot while a journal exists, so the units come up again here.
|
||||
if err := t.startUnits(ctx, j, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return t.remove()
|
||||
default:
|
||||
return t.rollback(ctx, j)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Transaction) promote(ctx context.Context, j journal) error {
|
||||
if err := t.setPhase(&j, phaseSwitching); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.switchOver(ctx, &j); err != nil {
|
||||
t.say("update to %s failed: %v; rolling back", j.Version, err)
|
||||
return t.rollback(ctx, j)
|
||||
}
|
||||
if j.Managed {
|
||||
if err := t.writeReport(j, "passed", false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
t.say("installed %s; restart the archie services to run it", j.Version)
|
||||
}
|
||||
if err := t.setPhase(&j, phaseCommitted); err != nil {
|
||||
return err
|
||||
}
|
||||
return t.remove()
|
||||
}
|
||||
|
||||
// switchOver replaces the executables and image policy, then restarts the
|
||||
// units in release order and requires the daemon to stay healthy.
|
||||
func (t *Transaction) switchOver(ctx context.Context, j *journal) error {
|
||||
if err := t.stopUnits(ctx, *j); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, unit := range j.Units {
|
||||
if err := os.Rename(t.staged(unit), filepath.Join(t.BinDir, unit)); err != nil {
|
||||
return fmt.Errorf("switch %s: %w", unit, err)
|
||||
}
|
||||
}
|
||||
if err := writeFileAtomic(filepath.Join(t.BinDir, agentVersionFile), []byte(j.Version+"\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syncDir(t.BinDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.setImage(ctx, *j, j.Image.Previous, j.Image.Candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
if !j.Managed {
|
||||
return nil
|
||||
}
|
||||
if err := t.setPhase(j, phaseObserving); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.startUnits(ctx, *j, stateStoreUnit); err != nil {
|
||||
return err
|
||||
}
|
||||
return t.observe(ctx, j.Version)
|
||||
}
|
||||
|
||||
// rollback is safe to repeat from any point: every step converges on the
|
||||
// previous state whether or not an earlier attempt got that far.
|
||||
func (t *Transaction) rollback(ctx context.Context, j journal) error {
|
||||
if err := t.setPhase(&j, phaseRollingBack); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.stopUnits(ctx, j); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
if err := t.restoreFiles(j); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
restored, err := t.restoreDatabase(ctx, j)
|
||||
if err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
started := ""
|
||||
if restored {
|
||||
// The snapshot predates the policy switch, so it holds the previous image.
|
||||
j.RestoredDatabase = true
|
||||
if err := t.setPhase(&j, phaseRollingBack); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
t.say("a migration ran: restored the database from %s; every write made after the update began is gone", t.snapshot())
|
||||
} else if err := t.setImage(ctx, j, j.Image.Candidate, j.Image.Previous); err != nil {
|
||||
return t.incomplete(err)
|
||||
} else {
|
||||
started = stateStoreUnit
|
||||
}
|
||||
if err := t.writeReport(j, "failed", true); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
if err := t.setPhase(&j, phaseRolledBack); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
if err := t.startUnits(ctx, j, started); err != nil {
|
||||
return t.incomplete(err)
|
||||
}
|
||||
return t.remove()
|
||||
}
|
||||
|
||||
// abandon discards a transaction that never switched anything.
|
||||
func (t *Transaction) abandon(ctx context.Context, j journal) error {
|
||||
t.say("update to %s was staged but never started; discarding it", j.Version)
|
||||
for _, unit := range j.Units {
|
||||
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := t.writeReport(j, "failed", true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.setPhase(&j, phaseRolledBack); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.startUnits(ctx, j, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return t.remove()
|
||||
}
|
||||
|
||||
func (t *Transaction) incomplete(err error) error {
|
||||
return fmt.Errorf("rollback incomplete, journal kept at %s for the next boot: %w", JournalPath(t.StateDir), err)
|
||||
}
|
||||
|
||||
func (t *Transaction) restoreFiles(j journal) error {
|
||||
previous := filepath.Join(JournalPath(t.StateDir), "previous")
|
||||
for _, unit := range j.Units {
|
||||
live := filepath.Join(t.BinDir, unit)
|
||||
saved := filepath.Join(previous, unit)
|
||||
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
|
||||
if err := os.Remove(live); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
} else if err := installFile(saved, live, 0o755); err != nil {
|
||||
return fmt.Errorf("restore %s: %w", unit, err)
|
||||
}
|
||||
if err := os.Remove(t.staged(unit)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
version := filepath.Join(t.BinDir, agentVersionFile)
|
||||
saved := filepath.Join(JournalPath(t.StateDir), agentVersionFile)
|
||||
if _, err := os.Stat(saved); errors.Is(err, os.ErrNotExist) {
|
||||
if err := os.Remove(version); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
} else if err := installFile(saved, version, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(t.BinDir)
|
||||
}
|
||||
|
||||
// restoreDatabase restores the pre-update snapshot only when the candidate
|
||||
// migrated the schema: the previous binaries refuse a newer schema, and data
|
||||
// written since the snapshot is otherwise worth keeping.
|
||||
func (t *Transaction) restoreDatabase(ctx context.Context, j journal) (bool, error) {
|
||||
if j.RestoredDatabase {
|
||||
return true, nil
|
||||
}
|
||||
schema, err := t.Database.SchemaVersion(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read schema version: %w", err)
|
||||
}
|
||||
if schema <= j.SchemaBefore {
|
||||
return false, nil
|
||||
}
|
||||
if err := t.Database.Restore(ctx, t.snapshot()); err != nil {
|
||||
return false, fmt.Errorf("restore database after migration %d -> %d: %w", j.SchemaBefore, schema, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// setImage retries while the State Store comes up.
|
||||
func (t *Transaction) setImage(ctx context.Context, j journal, expected, desired string) error {
|
||||
if j.Managed {
|
||||
if err := t.Units.Start(ctx, stateStoreUnit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
deadline := time.Now().Add(t.HealthTimeout)
|
||||
for {
|
||||
err := t.Policy.SetImage(ctx, expected, desired)
|
||||
if err == nil || time.Now().After(deadline) {
|
||||
return err
|
||||
}
|
||||
if err := sleep(ctx, t.Poll); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Transaction) observe(ctx context.Context, version string) error {
|
||||
deadline := time.Now().Add(t.HealthTimeout)
|
||||
for {
|
||||
reported, err := t.Health.Healthy(ctx)
|
||||
if err == nil && sameVersion(reported, version) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("archied never reported healthy on %s: %w", version, healthProblem(reported, err))
|
||||
}
|
||||
if err := sleep(ctx, t.Poll); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
t.say("archied is healthy on %s; watching it for %s", version, t.Observe)
|
||||
until := time.Now().Add(t.Observe)
|
||||
for time.Now().Before(until) {
|
||||
if err := sleep(ctx, t.Poll); err != nil {
|
||||
return err
|
||||
}
|
||||
reported, err := t.Health.Healthy(ctx)
|
||||
if err != nil || !sameVersion(reported, version) {
|
||||
return fmt.Errorf("archied became unhealthy while observed: %w", healthProblem(reported, err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func healthProblem(reported string, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("it reports version %q", reported)
|
||||
}
|
||||
|
||||
func sameVersion(left, right string) bool {
|
||||
l, ok := comparableVersion(left)
|
||||
r, _ := comparableVersion(right)
|
||||
return ok && l == r
|
||||
}
|
||||
|
||||
// startUnits restarts the units in release order, skipping one already started.
|
||||
func (t *Transaction) startUnits(ctx context.Context, j journal, skip string) error {
|
||||
if !j.Managed {
|
||||
return nil
|
||||
}
|
||||
for _, unit := range j.Units {
|
||||
if unit == skip {
|
||||
continue
|
||||
}
|
||||
if err := t.Units.Start(ctx, unit); err != nil {
|
||||
return fmt.Errorf("start %s: %w", unit, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stopUnits stops in reverse release order, so callers go before the services
|
||||
// they depend on.
|
||||
func (t *Transaction) stopUnits(ctx context.Context, j journal) error {
|
||||
if !j.Managed {
|
||||
return nil
|
||||
}
|
||||
for _, unit := range slices.Backward(j.Units) {
|
||||
if err := t.Units.Stop(ctx, unit); err != nil {
|
||||
return fmt.Errorf("stop %s: %w", unit, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Transaction) writeReport(j journal, health string, rolledBack bool) error {
|
||||
if j.Meta.ReportPath == "" {
|
||||
return nil
|
||||
}
|
||||
installed := map[string]string{ComponentDaemon: j.Version, ComponentAgent: j.Version}
|
||||
return WritePendingReport(j.Meta.ReportPath, Report{
|
||||
Channel: j.Meta.Channel, ChatID: j.Meta.ChatID, ThreadID: j.Meta.ThreadID,
|
||||
Previous: j.Previous, Installed: installed,
|
||||
HealthCheck: health, RolledBack: rolledBack, RestoredDatabase: j.RestoredDatabase,
|
||||
})
|
||||
}
|
||||
|
||||
func (t *Transaction) staged(unit string) string {
|
||||
return filepath.Join(t.BinDir, "."+unit+".new")
|
||||
}
|
||||
|
||||
func (t *Transaction) snapshot() string { return filepath.Join(t.StateDir, snapshotFile) }
|
||||
|
||||
func (t *Transaction) say(format string, args ...any) {
|
||||
if t.Progress != nil {
|
||||
t.Progress(fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Transaction) load() (journal, bool, error) {
|
||||
data, err := os.ReadFile(filepath.Join(JournalPath(t.StateDir), journalFile))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return journal{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return journal{}, false, fmt.Errorf("read update journal: %w", err)
|
||||
}
|
||||
var j journal
|
||||
if err := json.Unmarshal(data, &j); err != nil {
|
||||
return journal{}, false, fmt.Errorf("decode update journal: %w", err)
|
||||
}
|
||||
return j, true, nil
|
||||
}
|
||||
|
||||
func (t *Transaction) setPhase(j *journal, phase string) error {
|
||||
j.Phase = phase
|
||||
return writeJournal(JournalPath(t.StateDir), *j)
|
||||
}
|
||||
|
||||
func writeJournal(dir string, j journal) error {
|
||||
data, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFileAtomic(filepath.Join(dir, journalFile), data); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncDir(dir)
|
||||
}
|
||||
|
||||
// remove retires the journal by rename first, so a crash mid-delete never
|
||||
// leaves a partial journal behind.
|
||||
func (t *Transaction) remove() error {
|
||||
finished := filepath.Join(t.StateDir, "finished")
|
||||
if err := os.RemoveAll(finished); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(JournalPath(t.StateDir), finished); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syncDir(t.StateDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(finished)
|
||||
}
|
||||
|
||||
// lockState takes the update lock. Without wait it reports held=false at once
|
||||
// when another process owns it.
|
||||
func lockState(ctx context.Context, dir string, wait bool) (unlock func(), held bool, err error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
file, err := os.OpenFile(filepath.Join(dir, "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
deadline := time.Now().Add(lockWait)
|
||||
for {
|
||||
err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB)
|
||||
if err == nil {
|
||||
return func() { _ = file.Close() }, true, nil
|
||||
}
|
||||
if !errors.Is(err, unix.EWOULDBLOCK) || !wait || time.Now().After(deadline) {
|
||||
_ = file.Close()
|
||||
if errors.Is(err, unix.EWOULDBLOCK) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
if err := sleep(ctx, 100*time.Millisecond); err != nil {
|
||||
_ = file.Close()
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sleep(ctx context.Context, d time.Duration) error {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func syncDir(dir string) error {
|
||||
file, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
return file.Sync()
|
||||
}
|
||||
|
||||
// installFile copies src over dst through a synced temp file and rename, so
|
||||
// dst is always either the old or the new content.
|
||||
func installFile(src, dst string, mode os.FileMode) error {
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".tmp")
|
||||
if err := writeSynced(tmp, data, mode); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dst)
|
||||
}
|
||||
|
||||
func writeSynced(path string, data []byte, mode os.FileMode) error {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := file.Write(data); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Chmod(mode); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
package releaseupdate
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testUnits = []string{"archie-state-store", "archie-gateway", "archied"}
|
||||
|
||||
const (
|
||||
oldImage = "ghcr.io/example/archie-agent@sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
newImage = "ghcr.io/example/archie-agent@sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// fakeHost stands in for systemd, Docker, the State Store and the daemon. A
|
||||
// started State Store on a migrating release bumps the schema, as a real
|
||||
// candidate's migrations would.
|
||||
type fakeHost struct {
|
||||
bin string
|
||||
managed bool
|
||||
migrates bool
|
||||
unhealthy bool
|
||||
image string
|
||||
backedUp string
|
||||
schema int64
|
||||
pulled []string
|
||||
started []string
|
||||
restored bool
|
||||
}
|
||||
|
||||
func (h *fakeHost) Managed(context.Context) bool { return h.managed }
|
||||
|
||||
func (h *fakeHost) Start(_ context.Context, unit string) error {
|
||||
h.started = append(h.started, unit)
|
||||
if data, _ := os.ReadFile(filepath.Join(h.bin, unit)); h.migrates && unit == stateStoreUnit && bytes.Contains(data, []byte("2.0.0")) {
|
||||
h.schema = 2
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *fakeHost) Stop(context.Context, string) error { return nil }
|
||||
|
||||
func (h *fakeHost) Pull(_ context.Context, ref string) error {
|
||||
h.pulled = append(h.pulled, ref)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *fakeHost) Image(context.Context) (string, error) { return h.image, nil }
|
||||
|
||||
func (h *fakeHost) SetImage(_ context.Context, expected, desired string) error {
|
||||
if h.image != desired && h.image != expected {
|
||||
return errors.New("image policy changed")
|
||||
}
|
||||
h.image = desired
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *fakeHost) Backup(_ context.Context, out string) error {
|
||||
h.backedUp = h.image
|
||||
return os.WriteFile(out, []byte("snapshot"), 0o600)
|
||||
}
|
||||
|
||||
func (h *fakeHost) SchemaVersion(context.Context) (int64, error) { return h.schema, nil }
|
||||
|
||||
func (h *fakeHost) Restore(_ context.Context, from string) error {
|
||||
if _, err := os.Stat(from); err != nil {
|
||||
return err
|
||||
}
|
||||
h.restored, h.schema, h.image = true, 1, h.backedUp
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *fakeHost) Healthy(context.Context) (string, error) {
|
||||
if h.unhealthy {
|
||||
return "", errors.New("503 Service Unavailable")
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(h.bin, "archied"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.Fields(string(data))[3], nil
|
||||
}
|
||||
|
||||
// crash leaves the host as a watchdog killed during phase would have.
|
||||
func crash(t *testing.T, tx *Transaction, h *fakeHost, phase string) {
|
||||
t.Helper()
|
||||
j, found, err := tx.load()
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no journal to crash: %v", err)
|
||||
}
|
||||
switched := map[string]int{phasePrepared: 0, phaseSwitching: 1, phaseRolledBack: 0}
|
||||
count, partial := switched[phase]
|
||||
if !partial {
|
||||
count = len(j.Units)
|
||||
h.image = newImage
|
||||
}
|
||||
for _, unit := range j.Units[:count] {
|
||||
if err := os.Rename(tx.staged(unit), filepath.Join(tx.BinDir, unit)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if phase == phaseRolledBack {
|
||||
for _, unit := range j.Units {
|
||||
if err := os.Remove(tx.staged(unit)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if phase == phaseRollingBack {
|
||||
if err := installFile(filepath.Join(JournalPath(tx.StateDir), "previous", j.Units[0]), filepath.Join(tx.BinDir, j.Units[0]), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tx.setPhase(&j, phase); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransaction(t *testing.T) {
|
||||
type outcome struct {
|
||||
version, image string
|
||||
report string // health_check, or "" for no report
|
||||
restoredDatabase bool
|
||||
}
|
||||
old := outcome{version: "1.0.0", image: oldImage, report: "failed"}
|
||||
tests := []struct {
|
||||
name string
|
||||
host fakeHost
|
||||
candidate string // version the release's archied reports
|
||||
corrupt bool // SHA256SUMS does not match the zip
|
||||
crashAt string // phase the watchdog dies in, then boot recovers
|
||||
prepare string // Prepare refusal
|
||||
want outcome
|
||||
}{
|
||||
{name: "healthy release commits", host: fakeHost{managed: true}, want: outcome{version: "2.0.0", image: newImage, report: "passed"}},
|
||||
{name: "unhealthy release rolls back", host: fakeHost{managed: true, unhealthy: true}, want: old},
|
||||
{name: "unhealthy migrated release restores the database", host: fakeHost{managed: true, unhealthy: true, migrates: true}, want: outcome{version: "1.0.0", image: oldImage, report: "failed", restoredDatabase: true}},
|
||||
{name: "without systemd it installs without restarting", host: fakeHost{}, want: outcome{version: "2.0.0", image: newImage}},
|
||||
{name: "checksum mismatch is refused", host: fakeHost{managed: true}, corrupt: true, prepare: "checksum mismatch", want: outcome{version: "1.0.0", image: oldImage}},
|
||||
{name: "misreported candidate is refused", host: fakeHost{managed: true}, candidate: "1.9.0", prepare: "reports", want: outcome{version: "1.0.0", image: oldImage}},
|
||||
{name: "crash before the watchdog started is abandoned", host: fakeHost{managed: true}, crashAt: phasePrepared, want: old},
|
||||
{name: "crash while switching rolls back", host: fakeHost{managed: true}, crashAt: phaseSwitching, want: old},
|
||||
{name: "crash while observing rolls back", host: fakeHost{managed: true}, crashAt: phaseObserving, want: old},
|
||||
{name: "crash while rolling back finishes it", host: fakeHost{managed: true}, crashAt: phaseRollingBack, want: old},
|
||||
{name: "crash after commit keeps the release", host: fakeHost{managed: true}, crashAt: phaseCommitted, want: outcome{version: "2.0.0", image: newImage}},
|
||||
{name: "crash after rollback keeps the old release", host: fakeHost{managed: true}, crashAt: phaseRolledBack, want: outcome{version: "1.0.0", image: oldImage}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
host := tt.host
|
||||
host.bin, host.image, host.schema = filepath.Join(root, "bin"), oldImage, 1
|
||||
candidate := tt.candidate
|
||||
if candidate == "" {
|
||||
candidate = "2.0.0"
|
||||
}
|
||||
server := httptest.NewServer(releaseHandler(t, candidate, tt.corrupt))
|
||||
defer server.Close()
|
||||
for _, unit := range testUnits {
|
||||
writeTestFile(t, filepath.Join(host.bin, unit), versionScript(unit, "1.0.0"))
|
||||
}
|
||||
self := filepath.Join(root, "self")
|
||||
writeTestFile(t, self, "frozen watchdog")
|
||||
report := filepath.Join(root, "report.json")
|
||||
tx := &Transaction{
|
||||
StateDir: filepath.Join(root, "state"), BinDir: host.bin,
|
||||
Units: &host, Images: &host, Policy: &host, Database: &host, Health: &host,
|
||||
HealthTimeout: 20 * time.Millisecond, Observe: 5 * time.Millisecond, Poll: time.Millisecond,
|
||||
}
|
||||
|
||||
_, err := tx.Prepare(t.Context(), PrepareRequest{Version: "2.0.0", ReleaseURL: server.URL, Self: self, Meta: InstallMeta{Channel: "webui", ReportPath: report}})
|
||||
switch {
|
||||
case tt.prepare != "":
|
||||
if err == nil || !strings.Contains(err.Error(), tt.prepare) {
|
||||
t.Fatalf("Prepare error = %v, want %q", err, tt.prepare)
|
||||
}
|
||||
case err != nil:
|
||||
t.Fatalf("Prepare: %v", err)
|
||||
case tt.crashAt != "":
|
||||
crash(t, tx, &host, tt.crashAt)
|
||||
err = tx.Run(t.Context(), true)
|
||||
default:
|
||||
err = tx.Run(t.Context(), false)
|
||||
}
|
||||
if tt.prepare == "" && err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
|
||||
for _, unit := range testUnits {
|
||||
data, _ := os.ReadFile(filepath.Join(host.bin, unit))
|
||||
if !strings.Contains(string(data), " "+tt.want.version) {
|
||||
t.Errorf("%s = %q, want %s", unit, data, tt.want.version)
|
||||
}
|
||||
if _, err := os.Stat(tx.staged(unit)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Errorf("%s is still staged", unit)
|
||||
}
|
||||
}
|
||||
if host.image != tt.want.image {
|
||||
t.Errorf("image = %s, want %s", host.image, tt.want.image)
|
||||
}
|
||||
if _, err := os.Stat(JournalPath(tx.StateDir)); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Error("journal left behind")
|
||||
}
|
||||
if host.restored != tt.want.restoredDatabase {
|
||||
t.Errorf("database restored = %v, want %v", host.restored, tt.want.restoredDatabase)
|
||||
}
|
||||
if host.managed && tt.prepare == "" && !slices.Contains(host.started, "archied") {
|
||||
t.Errorf("archied never started: %v", host.started)
|
||||
}
|
||||
if !host.managed && len(host.started) > 0 {
|
||||
t.Errorf("started %v without a service manager", host.started)
|
||||
}
|
||||
got, found, err := ReadPendingReport(report)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wantReport := tt.want.report != "" && tt.crashAt != phaseCommitted && tt.crashAt != phaseRolledBack; found != wantReport ||
|
||||
found && (got.HealthCheck != tt.want.report || got.RolledBack != (tt.want.report == "failed") || got.RestoredDatabase != tt.want.restoredDatabase) {
|
||||
t.Errorf("report = %+v (found %v), want %+v", got, found, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// releaseHandler serves one release's zip and SHA256SUMS the way GitHub
|
||||
// release downloads are laid out.
|
||||
func releaseHandler(t *testing.T, candidate string, corrupt bool) http.Handler {
|
||||
t.Helper()
|
||||
name := "archie-core-archied-v2.0.0-linux-amd64"
|
||||
var archive bytes.Buffer
|
||||
writer := zip.NewWriter(&archive)
|
||||
add := func(path, body string) {
|
||||
header := &zip.FileHeader{Name: name + "/" + path, Method: zip.Deflate}
|
||||
header.SetMode(0o755)
|
||||
file, err := writer.CreateHeader(header)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := file.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, unit := range testUnits {
|
||||
version := "2.0.0"
|
||||
if unit == "archied" {
|
||||
version = candidate
|
||||
}
|
||||
add(unit, versionScript(unit, version))
|
||||
}
|
||||
add("release.json", fmt.Sprintf(`{"required_topology":{"units":["%s"]},"agent_image":%q}`, strings.Join(testUnits, `","`), newImage))
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(archive.Bytes())
|
||||
if corrupt {
|
||||
sum[0] ^= 0xff
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v2.0.0/"+name+".zip", func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(archive.Bytes()) })
|
||||
mux.HandleFunc("/v2.0.0/SHA256SUMS", func(w http.ResponseWriter, _ *http.Request) { fmt.Fprintf(w, "%x %s.zip\n", sum, name) })
|
||||
return mux
|
||||
}
|
||||
|
||||
func versionScript(unit, version string) string {
|
||||
return fmt.Sprintf("#!/bin/sh\n# %s %s\necho '%s %s'\n", unit, version, unit, version)
|
||||
}
|
||||
|
||||
func writeTestFile(t *testing.T, path, data string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(data), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -1,284 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# archie-update-install -- component-aware update installation adapter.
|
||||
#
|
||||
# CommandInstaller passes the freshly revalidated, approved snapshot through
|
||||
# ARCHIE_UPDATE_{DAEMON,AGENT}_{PREVIOUS,VERSION}. A VERSION is non-empty only
|
||||
# when that component changed in the approved snapshot. This script must never
|
||||
# rediscover a broader plan after approval.
|
||||
set -euo pipefail
|
||||
|
||||
json_escape() {
|
||||
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
||||
}
|
||||
|
||||
SOURCE_URL="${ARCHIE_SOURCE_URL:-https://github.com/samcharles93/archie-core.git}"
|
||||
BIN_DIR="${ARCHIE_BIN_DIR:-$HOME/.local/bin}"
|
||||
CONFIG_PATH="${ARCHIE_CONFIG_PATH:-${XDG_CONFIG_HOME:-$HOME/.config}/archie/config.toml}"
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
# The units load this file; a run from a shell needs it too, for PGPASSWORD.
|
||||
ENV_FILE="$(dirname "$CONFIG_PATH")/env"
|
||||
if [ -r "$ENV_FILE" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
|
||||
# State for the managed image, independent of archied's daemon build stamp.
|
||||
AGENT_VERSION_FILE="${ARCHIE_AGENT_VERSION_FILE:-$BIN_DIR/archie-agent.version}"
|
||||
DB_SNAPSHOT="${ARCHIE_DB_SNAPSHOT:-$BIN_DIR/archie-db.pre-update.dump}"
|
||||
# The address the daemon dials for its state, so the watchdog can wait for it to
|
||||
# accept connections before it starts archied. Read with a narrow awk: the
|
||||
# daemon stays the authority on validation, and this script deliberately
|
||||
# understands one scalar rather than parsing TOML. Scoped to [services.state] so the Gateway's own target cannot be
|
||||
# mistaken for it, and exact on the key so target_token is not.
|
||||
configured_state_store_target() {
|
||||
awk '
|
||||
/^[[:space:]]*\[services\.state\][[:space:]]*$/ { inside = 1; next }
|
||||
/^[[:space:]]*\[/ { inside = 0 }
|
||||
inside && /^[[:space:]]*target[[:space:]]*=/ {
|
||||
value = $0
|
||||
sub(/^[[:space:]]*target[[:space:]]*=[[:space:]]*/, "", value)
|
||||
sub(/[[:space:]]*(#.*)?$/, "", value)
|
||||
if (value ~ /^".*"$/) { sub(/^"/, "", value); sub(/"$/, "", value); print value }
|
||||
else if (length(value) >= 2 && substr(value, 1, 1) == sprintf("%c", 39) && substr(value, length(value), 1) == sprintf("%c", 39)) print substr(value, 2, length(value) - 2)
|
||||
exit
|
||||
}
|
||||
' "$CONFIG_PATH" 2>/dev/null || true
|
||||
}
|
||||
|
||||
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-$(configured_state_store_target)}"
|
||||
|
||||
GATEWAY_SERVICES="archie-state-store archie-gateway archie-ui archie-messaging archied"
|
||||
GATEWAY_BINARIES="$GATEWAY_SERVICES"
|
||||
|
||||
SOURCE_BUILD="${ARCHIE_UPDATE_SOURCE_BUILD:-0}"
|
||||
AUTO=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--from-source) SOURCE_BUILD=1 ;;
|
||||
--auto) AUTO=1 ;;
|
||||
*) echo "unknown option: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$AUTO" = 1 ] && [ -z "${ARCHIE_UPDATE_DAEMON_VERSION:-}${ARCHIE_UPDATE_AGENT_VERSION:-}" ]; then
|
||||
snapshot="$("$BIN_DIR/archied" update check)"
|
||||
export ARCHIE_UPDATE_DAEMON_PREVIOUS="$(jq -r '.Components[] | select(.ID == "daemon") | .Installed' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_DAEMON_VERSION="$(jq -r '.Components[] | select(.ID == "daemon") | .Available' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_AGENT_PREVIOUS="$(jq -r '.Components[] | select(.ID == "agent") | .Installed' <<< "$snapshot")"
|
||||
export ARCHIE_UPDATE_AGENT_VERSION="$(jq -r '.Components[] | select(.ID == "agent") | .Available' <<< "$snapshot")"
|
||||
fi
|
||||
|
||||
PREV_GW="${ARCHIE_UPDATE_DAEMON_PREVIOUS:-}"
|
||||
PREV_RT="${ARCHIE_UPDATE_AGENT_PREVIOUS:-}"
|
||||
GW_TARGET="${ARCHIE_UPDATE_DAEMON_VERSION:-}"
|
||||
RT_TARGET="${ARCHIE_UPDATE_AGENT_VERSION:-}"
|
||||
GW="$PREV_GW"
|
||||
RT="$PREV_RT"
|
||||
components=""
|
||||
|
||||
if [ -n "$GW_TARGET" ] && [ "$GW_TARGET" != "$PREV_GW" ]; then
|
||||
GW="$GW_TARGET"
|
||||
components="daemon"
|
||||
fi
|
||||
if [ -n "$RT_TARGET" ] && [ "$RT_TARGET" != "$PREV_RT" ]; then
|
||||
case "$RT_TARGET" in
|
||||
dev|unknown) echo "refusing to build a released runtime image with version '$RT_TARGET'" >&2; exit 1 ;;
|
||||
esac
|
||||
RT="$RT_TARGET"
|
||||
components="${components:+$components,}agent"
|
||||
fi
|
||||
|
||||
verify_installed_versions() {
|
||||
reference="$1"
|
||||
problems=""
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
[ -f "$BIN_DIR/$cmd" ] || continue
|
||||
reported="$("$BIN_DIR/$cmd" -version 2>/dev/null | awk -v want="$cmd" '$1 == want { print $2; exit }')"
|
||||
[ -n "$reported" ] || reported="unknown"
|
||||
if [ "$reported" != "$reference" ]; then
|
||||
problems="${problems:+$problems }$cmd=$reported"
|
||||
fi
|
||||
done
|
||||
if [ -n "$problems" ]; then
|
||||
echo "refusing: installed binaries disagree with $reference: $problems" >&2
|
||||
echo "each host binary must report the release it was built from; reinstall the outliers from archied/v$reference, or approve an update that rebuilds them" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -z "$components" ]; then
|
||||
reference="$PREV_GW"
|
||||
if [ -z "$reference" ]; then
|
||||
reference="$("$BIN_DIR/archied" -version 2>/dev/null | awk '$1 == "archied" { print $2; exit }')"
|
||||
fi
|
||||
if [ -n "$reference" ] && ! verify_installed_versions "$reference"; then
|
||||
exit 1
|
||||
fi
|
||||
echo '==> no approved component changes; nothing to install'
|
||||
echo 'ARCHIE_UPDATE_RESULT {"previous":{},"installed":{},"components":[],"restart_requested":false}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
valid_release_version() {
|
||||
[[ "$1" =~ ^[0-9]+\.[0-9]+\.[0-9]+([+-][0-9A-Za-z.-]+)?$ ]]
|
||||
}
|
||||
|
||||
for target in "$GW_TARGET" "$RT_TARGET"; do
|
||||
if [ -n "$target" ] && ! valid_release_version "$target"; then
|
||||
echo "refusing non-release component version '$target'" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
|
||||
mkdir -p "$STATE_DIR"
|
||||
chmod 700 "$STATE_DIR"
|
||||
exec 9>"$STATE_DIR/lock"
|
||||
if ! flock -n 9; then
|
||||
echo "update locked by $(cat "$STATE_DIR/holder" 2>/dev/null); journal: $STATE_DIR/transaction" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'installer PID %s\n' "$$" > "$STATE_DIR/holder"
|
||||
if [ -d "$STATE_DIR/transaction" ]; then
|
||||
echo "unfinished update: run $STATE_DIR/transaction/watchdog --recover before retrying" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
source_dir="$tmp/source"
|
||||
release_dir="$tmp/release"
|
||||
mkdir -p "$release_dir"
|
||||
release_version="${GW_TARGET:-$RT_TARGET}"
|
||||
if [ -n "$GW_TARGET" ] && [ -n "$RT_TARGET" ] && [ "$GW_TARGET" != "$RT_TARGET" ]; then
|
||||
echo "daemon and agent must name the same release" >&2; exit 1
|
||||
fi
|
||||
if [ "$SOURCE_BUILD" = 1 ]; then
|
||||
tag="v$release_version"
|
||||
git ls-remote --exit-code --tags "$SOURCE_URL" "refs/tags/$tag" >/dev/null || tag="archied/v$release_version"
|
||||
git clone --quiet --depth=1 --branch "$tag" "$SOURCE_URL" "$source_dir"
|
||||
cp "$source_dir/release.json" "$release_dir/"
|
||||
GATEWAY_BINARIES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
|
||||
cp "$source_dir"/scripts/archie-update-* "$release_dir/"
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
[[ "$cmd" =~ ^archie[a-z0-9-]*$ ]] || { echo "invalid release unit: $cmd" >&2; exit 1; }
|
||||
(cd "$source_dir" && go build -ldflags "-X github.com/samcharles93/archie-core/internal/installtype.buildType=binary -X github.com/samcharles93/archie-core/internal/buildinfo.Version=$GW -X github.com/samcharles93/archie-core/internal/buildinfo.Runtime=$RT" -o "$release_dir/$cmd" "./cmd/$cmd")
|
||||
done
|
||||
fi
|
||||
else
|
||||
case "$(uname -s)/$(uname -m)" in
|
||||
Linux/x86_64) platform=linux-amd64 ;;
|
||||
*) echo "no published artifact for $(uname -s)/$(uname -m)" >&2; exit 1 ;;
|
||||
esac
|
||||
name="archie-core-archied-v$release_version-$platform"
|
||||
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/v$release_version"
|
||||
if ! curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"; then
|
||||
base="${ARCHIE_RELEASE_BASE_URL:-https://github.com/samcharles93/archie-core/releases/download}/archied/v$release_version"
|
||||
curl -fLSs --max-time 300 "$base/$name.zip" -o "$tmp/$name.zip"
|
||||
fi
|
||||
curl -fLSs --max-time 60 "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"
|
||||
expected="$(awk -v name="$name.zip" '$2 == name {print $1}' "$tmp/SHA256SUMS")"
|
||||
[[ "$expected" =~ ^[a-fA-F0-9]{64}$ ]] || { echo "missing or ambiguous archive checksum" >&2; exit 1; }
|
||||
printf '%s %s\n' "$expected" "$tmp/$name.zip" | sha256sum --check --status || { echo "release checksum mismatch" >&2; exit 1; }
|
||||
# Reject paths outside the single distribution directory before extracting.
|
||||
while IFS= read -r entry; do
|
||||
case "$entry" in "$name/"*) ;; *) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
|
||||
case "/$entry/" in */../*|*/./*|*\\*) echo "unsafe archive path: $entry" >&2; exit 1 ;; esac
|
||||
done < <(unzip -Z1 "$tmp/$name.zip")
|
||||
if unzip -Z -l "$tmp/$name.zip" | awk '$1 ~ /^l/ {found=1} END {exit !found}'; then
|
||||
echo "release contains symlinks" >&2; exit 1
|
||||
fi
|
||||
unzip -q "$tmp/$name.zip" -d "$tmp/unpacked"
|
||||
if find "$tmp/unpacked" -type l | grep -q .; then echo "release contains symlinks" >&2; exit 1; fi
|
||||
cp -a "$tmp/unpacked/$name/." "$release_dir/"
|
||||
fi
|
||||
|
||||
# Observation and config parsing belong to the daemon, including on old hosts.
|
||||
observer="$release_dir/archied"
|
||||
[ -x "$observer" ] || observer="$BIN_DIR/archied"
|
||||
"$observer" update-topology -config "$CONFIG_PATH" -release "$release_dir/release.json"
|
||||
GATEWAY_SERVICES="$(jq -er '.required_topology.units | join(" ")' "$release_dir/release.json")"
|
||||
GATEWAY_BINARIES="$GATEWAY_SERVICES"
|
||||
|
||||
if [[ ",$components," == *,agent,* ]]; then
|
||||
if [ "$SOURCE_BUILD" = 1 ]; then
|
||||
docker build --build-arg "RUNTIME_VERSION=$RT" --iidfile "$tmp/image-id" "$source_dir"
|
||||
AGENT_IMAGE="$(cat "$tmp/image-id")"
|
||||
else
|
||||
AGENT_IMAGE="$(jq -er '.agent_image' "$release_dir/release.json")"
|
||||
[[ "$AGENT_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]] || { echo "refusing agent update: release has no digest-pinned image" >&2; exit 1; }
|
||||
docker pull "$AGENT_IMAGE"
|
||||
fi
|
||||
fi
|
||||
if [ "$AUTO" = 1 ]; then
|
||||
status="$(curl -fsS --max-time 10 "${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}/health/detailed")"
|
||||
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || { echo "refusing auto-update: daemon has work in flight or cannot report it" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
built=""
|
||||
if [[ ",$components," == *,daemon,* ]]; then
|
||||
for cmd in $GATEWAY_BINARIES; do
|
||||
reported="$("$release_dir/$cmd" -version | awk -v cmd="$cmd" '$1 == cmd {print $2; exit}')"
|
||||
[ "$reported" = "$GW" ] || { echo "candidate $cmd reports $reported, expected $GW" >&2; exit 1; }
|
||||
done
|
||||
mkdir -p "$(dirname "$DB_SNAPSHOT")"
|
||||
"$BIN_DIR/archie-state-store" backup -config "$CONFIG_PATH" -out "$DB_SNAPSHOT"
|
||||
built="$GATEWAY_BINARIES archie-update-install archie-update-watchdog"
|
||||
fi
|
||||
|
||||
# Freeze recovery code and every prior file before publishing the intent.
|
||||
journal="$STATE_DIR/preparing"
|
||||
rm -rf "$journal"
|
||||
mkdir -p "$journal/previous"
|
||||
cp "$release_dir/archie-update-watchdog" "$journal/watchdog"
|
||||
cp "$observer" "$journal/helper"
|
||||
if [[ ",$components," == *,agent,* ]]; then
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -prepare "$AGENT_IMAGE"
|
||||
fi
|
||||
chmod 700 "$journal/watchdog"
|
||||
for cmd in $built; do
|
||||
if [ -f "$BIN_DIR/$cmd" ]; then cp -p "$BIN_DIR/$cmd" "$journal/previous/$cmd"; fi
|
||||
install -m755 "$release_dir/$cmd" "$BIN_DIR/.$cmd.new"
|
||||
done
|
||||
if [ -f "$AGENT_VERSION_FILE" ]; then cp -p "$AGENT_VERSION_FILE" "$journal/agent.version"; fi
|
||||
# No unit/config changes are made: topology migration is explicitly refused.
|
||||
export ARCHIE_UPDATE_AUTO="$AUTO"
|
||||
export ARCHIE_UPDATE_STATE_DIR="$STATE_DIR"
|
||||
export ARCHIE_BIN_DIR="$BIN_DIR" ARCHIE_CONFIG_PATH="$CONFIG_PATH"
|
||||
export ARCHIE_UPDATE_COMPONENTS="$components" ARCHIE_UPDATE_UNITS="$GATEWAY_SERVICES" ARCHIE_UPDATE_BINARIES="$built"
|
||||
export ARCHIE_UPDATE_PREVIOUS_GATEWAY="$PREV_GW" ARCHIE_UPDATE_PREVIOUS_RUNTIME="$PREV_RT"
|
||||
export ARCHIE_UPDATE_INSTALLED_GATEWAY="$GW" ARCHIE_UPDATE_INSTALLED_RUNTIME="$RT"
|
||||
export ARCHIE_AGENT_VERSION_FILE="$AGENT_VERSION_FILE" ARCHIE_DB_SNAPSHOT="$DB_SNAPSHOT"
|
||||
export ARCHIE_UPDATE_STATE_STORE_TARGET="$STATE_STORE_TARGET"
|
||||
# Bash's quoted declarations preserve spaces without interpreting journal values.
|
||||
while IFS= read -r var; do declare -p "$var"; done < <(compgen -e | grep -E '^ARCHIE_(UPDATE_|BIN_DIR$|CONFIG_PATH$|HEALTH_URL$|AGENT_VERSION_FILE$|DB_SNAPSHOT$)') > "$journal/environment"
|
||||
printf 'prepared\n' > "$journal/phase"
|
||||
sync -f "$BIN_DIR"
|
||||
sync -f "$journal"
|
||||
mv "$journal" "$STATE_DIR/transaction"
|
||||
sync -f "$STATE_DIR"
|
||||
journal="$STATE_DIR/transaction"
|
||||
# The independent watchdog holds the lock and owns promotion, restart and rollback.
|
||||
systemd-run --user --unit=archied-update-restart --collect --quiet \
|
||||
--setenv=ARCHIE_UPDATE_STATE_DIR="$STATE_DIR" "$journal/watchdog"
|
||||
case "$components" in
|
||||
daemon)
|
||||
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\"}"
|
||||
installed="{\"daemon\":\"$(json_escape "$GW")\"}"
|
||||
results='[{"id":"daemon","label":"daemon","status":"updated"}]'
|
||||
;;
|
||||
agent)
|
||||
previous="{\"agent\":\"$(json_escape "$PREV_RT")\"}"
|
||||
installed="{\"agent\":\"$(json_escape "$RT")\"}"
|
||||
results='[{"id":"agent","label":"agent","status":"updated"}]'
|
||||
;;
|
||||
daemon,agent)
|
||||
previous="{\"daemon\":\"$(json_escape "$PREV_GW")\",\"agent\":\"$(json_escape "$PREV_RT")\"}"
|
||||
installed="{\"daemon\":\"$(json_escape "$GW")\",\"agent\":\"$(json_escape "$RT")\"}"
|
||||
results='[{"id":"daemon","label":"daemon","status":"updated"},{"id":"agent","label":"agent","status":"updated"}]'
|
||||
;;
|
||||
esac
|
||||
echo "ARCHIE_UPDATE_RESULT {\"previous\":$previous,\"installed\":$installed,\"components\":$results,\"restart_requested\":true}"
|
||||
@@ -1,265 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# archie-update-watchdog -- restarts archied after an update, verifies health,
|
||||
# and rolls back only the components named by ARCHIE_UPDATE_COMPONENTS.
|
||||
#
|
||||
# Runtime-only updates still restart archied: the restart is the existing
|
||||
# phase-2 handoff that makes the daemon consume and deliver the health report.
|
||||
# It does not rebuild or reinstall the gateway binary.
|
||||
set -uo pipefail
|
||||
|
||||
STATE_DIR="${ARCHIE_UPDATE_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/archie/update}"
|
||||
journal="$STATE_DIR/transaction"
|
||||
mkdir -p "$STATE_DIR"
|
||||
exec 9>"$STATE_DIR/lock"
|
||||
if [ "${1:-}" = --recover ]; then
|
||||
flock -n 9 || exit 0 # A live installer/watchdog owns recovery.
|
||||
else
|
||||
flock -w 60 9 || { echo "update locked by $(cat "$STATE_DIR/holder")" >&2; exit 1; }
|
||||
fi
|
||||
[ -d "$journal" ] || exit 0
|
||||
printf 'watchdog PID %s\n' "$$" > "$STATE_DIR/holder"
|
||||
source "$journal/environment"
|
||||
BIN_DIR="$ARCHIE_BIN_DIR"
|
||||
CONFIG_PATH="$ARCHIE_CONFIG_PATH"
|
||||
DB_SNAPSHOT="$ARCHIE_DB_SNAPSHOT"
|
||||
AGENT_VERSION_FILE="$ARCHIE_AGENT_VERSION_FILE"
|
||||
HEALTH_URL="${ARCHIE_HEALTH_URL:-http://127.0.0.1:8485}"
|
||||
HEALTH_URL="${HEALTH_URL%/}"
|
||||
REPORT_PATH="${ARCHIE_UPDATE_REPORT_PATH:-}"
|
||||
HEALTH_TIMEOUT_SECONDS="${ARCHIE_UPDATE_HEALTH_TIMEOUT:-60}"
|
||||
OBSERVE_SECONDS="${ARCHIE_UPDATE_OBSERVE_SECONDS:-120}"
|
||||
POLL_INTERVAL_SECONDS="${ARCHIE_UPDATE_POLL_INTERVAL:-3}"
|
||||
COMPONENTS="$ARCHIE_UPDATE_COMPONENTS"
|
||||
UNITS="$ARCHIE_UPDATE_UNITS"
|
||||
BINARIES="$ARCHIE_UPDATE_BINARIES"
|
||||
STATE_STORE_TARGET="${ARCHIE_UPDATE_STATE_STORE_TARGET:-}"
|
||||
STATE_STORE_TIMEOUT_SECONDS="${ARCHIE_UPDATE_STATE_STORE_TIMEOUT:-30}"
|
||||
|
||||
phase() {
|
||||
printf '%s\n' "$1" > "$journal/phase.new" && sync -f "$journal" &&
|
||||
mv "$journal/phase.new" "$journal/phase" && sync -f "$journal"
|
||||
}
|
||||
|
||||
json_escape() {
|
||||
printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'
|
||||
}
|
||||
|
||||
report_versions() {
|
||||
case "$COMPONENTS" in
|
||||
daemon)
|
||||
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\"}"
|
||||
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\"}"
|
||||
;;
|
||||
agent)
|
||||
previous_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
||||
installed_json="{\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
||||
;;
|
||||
daemon,agent)
|
||||
previous_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_PREVIOUS_RUNTIME:-unknown}")\"}"
|
||||
installed_json="{\"daemon\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_GATEWAY:-unknown}")\",\"agent\":\"$(json_escape "${ARCHIE_UPDATE_INSTALLED_RUNTIME:-unknown}")\"}"
|
||||
;;
|
||||
*)
|
||||
echo "invalid ARCHIE_UPDATE_COMPONENTS: $COMPONENTS" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
write_report() {
|
||||
local health_check="$1" rolled_back="$2"
|
||||
[ -z "$REPORT_PATH" ] && return 0
|
||||
report_versions || return 1
|
||||
mkdir -p "$(dirname "$REPORT_PATH")"
|
||||
tmp="$(mktemp "$(dirname "$REPORT_PATH")/.update-report-XXXXXX")"
|
||||
cat >"$tmp" <<EOF
|
||||
{"channel":"$(json_escape "${ARCHIE_UPDATE_CHANNEL:-}")","chat_id":${ARCHIE_UPDATE_CHAT_ID:-0},"thread_id":${ARCHIE_UPDATE_THREAD_ID:-0},"previous":$previous_json,"installed":$installed_json,"health_check":"$health_check","rolled_back":$rolled_back}
|
||||
EOF
|
||||
chmod 600 "$tmp"
|
||||
mv -f "$tmp" "$REPORT_PATH"
|
||||
}
|
||||
|
||||
# LAST_HEALTH is the last report healthy saw, printed when the update is
|
||||
# given up so the log says which component failed.
|
||||
LAST_HEALTH=""
|
||||
healthy() {
|
||||
local report
|
||||
if ! report="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")"; then
|
||||
LAST_HEALTH="$(curl -sS --max-time 10 "$HEALTH_URL/health/detailed" 2>&1)"
|
||||
return 1
|
||||
fi
|
||||
LAST_HEALTH="$report"
|
||||
[ "$(jq -r '.version' <<< "$report")" = "$ARCHIE_UPDATE_INSTALLED_GATEWAY" ] &&
|
||||
[ "$(jq -r '.status' <<< "$report")" = ok ]
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local deadline=$((SECONDS + HEALTH_TIMEOUT_SECONDS))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if healthy; then
|
||||
deadline=$((SECONDS + OBSERVE_SECONDS))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
healthy || { echo "==> unhealthy while observing: $LAST_HEALTH"; return 1; }
|
||||
done
|
||||
return 0
|
||||
fi
|
||||
sleep "$POLL_INTERVAL_SECONDS"
|
||||
done
|
||||
echo "==> never became healthy; last report: $LAST_HEALTH"
|
||||
return 1
|
||||
}
|
||||
|
||||
manage_unit() { systemctl --user "$1" "$2.service"; }
|
||||
|
||||
# port_accepts reports whether host:port accepts a connection. bash's /dev/tcp
|
||||
# costs no process and no dependency, and accepting is exactly the signal this
|
||||
# needs: the failure it guards against is `dial tcp ...: connect: connection
|
||||
# refused`, which is the port not being bound at all.
|
||||
port_accepts() {
|
||||
(exec 3<>"/dev/tcp/${1%:*}/${1##*:}") 2>/dev/null
|
||||
}
|
||||
|
||||
# Wait for the dependency before starting its callers.
|
||||
wait_for_state_store() {
|
||||
[ -n "$STATE_STORE_TARGET" ] || {
|
||||
echo "==> no State Store target known; starting the rest without waiting for it" >&2
|
||||
return 0
|
||||
}
|
||||
local deadline=$((SECONDS + STATE_STORE_TIMEOUT_SECONDS))
|
||||
while [ "$SECONDS" -lt "$deadline" ]; do
|
||||
if port_accepts "$STATE_STORE_TARGET"; then
|
||||
echo "==> $STATE_STORE_TARGET accepts connections"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "==> $STATE_STORE_TARGET did not accept connections within ${STATE_STORE_TIMEOUT_SECONDS}s; starting the rest anyway" >&2
|
||||
return 0
|
||||
}
|
||||
|
||||
# start_units brings the deployment up in dependency order: the State Store
|
||||
# before anything that dials it, the Gateway before archied and the UI. The order
|
||||
# sequences the calls; wait_for_state_store is what makes it mean "ready".
|
||||
start_units() {
|
||||
local unit rc=0
|
||||
for unit in $UNITS; do
|
||||
echo "==> restarting $unit.service"
|
||||
manage_unit restart "$unit" || rc=1
|
||||
if [ "$unit" = archie-state-store ]; then
|
||||
wait_for_state_store
|
||||
fi
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
# stop_units tears down in reverse, so consumers are gone before the services
|
||||
# they depend on.
|
||||
stop_units() {
|
||||
local unit ordered="" rc=0
|
||||
for unit in $UNITS; do
|
||||
ordered="$unit${ordered:+ $ordered}"
|
||||
done
|
||||
for unit in $ordered; do
|
||||
manage_unit stop "$unit" || rc=1
|
||||
done
|
||||
return $rc
|
||||
}
|
||||
|
||||
rollback_files() {
|
||||
local cmd
|
||||
for cmd in $BINARIES; do
|
||||
if [ -f "$journal/previous/$cmd" ]; then
|
||||
cp -p "$journal/previous/$cmd" "$BIN_DIR/.$cmd.rollback" &&
|
||||
mv -f "$BIN_DIR/.$cmd.rollback" "$BIN_DIR/$cmd" || return 1
|
||||
else
|
||||
rm -f "$BIN_DIR/$cmd" || return 1
|
||||
fi
|
||||
rm -f "$BIN_DIR/.$cmd.new"
|
||||
done
|
||||
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
||||
if [ -f "$journal/agent.version" ]; then
|
||||
cp -p "$journal/agent.version" "$AGENT_VERSION_FILE.new" &&
|
||||
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
||||
else
|
||||
rm -f "$AGENT_VERSION_FILE" || return 1
|
||||
fi
|
||||
fi
|
||||
sync -f "$BIN_DIR"
|
||||
}
|
||||
|
||||
finish() {
|
||||
phase "$1" || return 1
|
||||
# A terminal journal is harmless if power fails before removal.
|
||||
rm -rf "$STATE_DIR/finished"
|
||||
mv "$journal" "$STATE_DIR/finished" && sync -f "$STATE_DIR" || return 1
|
||||
rm -rf "$STATE_DIR/finished"
|
||||
}
|
||||
|
||||
rollback() {
|
||||
phase rolling-back || return 1
|
||||
stop_units && rollback_files || return 1
|
||||
if [ -f "$journal/image.json" ]; then
|
||||
manage_unit start archie-state-store && wait_for_state_store &&
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" -rollback || return 1
|
||||
fi
|
||||
phase rolled-back || return 1
|
||||
start_units || return 1
|
||||
write_report failed true || return 1
|
||||
finish rolled-back
|
||||
}
|
||||
|
||||
promote() {
|
||||
if [ "${ARCHIE_UPDATE_AUTO:-0}" = 1 ]; then
|
||||
local status
|
||||
status="$(curl -fsS --max-time 10 "$HEALTH_URL/health/detailed")" || return 2
|
||||
[ "$(jq -r '.in_flight' <<< "$status")" = 0 ] || return 2
|
||||
fi
|
||||
phase switching || return 1
|
||||
stop_units || return 1
|
||||
for cmd in $BINARIES; do
|
||||
mv -f "$BIN_DIR/.$cmd.new" "$BIN_DIR/$cmd" || return 1
|
||||
done
|
||||
if [[ ",$COMPONENTS," == *,agent,* ]]; then
|
||||
printf '%s\n' "$ARCHIE_UPDATE_INSTALLED_RUNTIME" > "$AGENT_VERSION_FILE.new" &&
|
||||
mv -f "$AGENT_VERSION_FILE.new" "$AGENT_VERSION_FILE" || return 1
|
||||
fi
|
||||
sync -f "$BIN_DIR" || return 1
|
||||
if [ -f "$journal/image.json" ]; then
|
||||
manage_unit start archie-state-store && wait_for_state_store &&
|
||||
"$journal/helper" update-image -config "$CONFIG_PATH" -journal "$journal" || return 1
|
||||
fi
|
||||
phase observing && start_units && wait_healthy
|
||||
}
|
||||
|
||||
# explain_database_recovery states what a daemon rollback does not do. The
|
||||
# PostgreSQL database is never restored automatically: goose Down drops tables,
|
||||
# and a restore is destructive, loses every write after the snapshot and needs
|
||||
# every service stopped, so it is an operator's decision, not this script's.
|
||||
explain_database_recovery() {
|
||||
echo "==> the PostgreSQL database was NOT rolled back; schema migrations are never undone automatically" >&2
|
||||
echo "==> if the previous release refuses the migrated schema, stop every archie service and restore the pre-update snapshot (every write made after it is lost):" >&2
|
||||
echo "==> archie-state-store restore -config $CONFIG_PATH -from $DB_SNAPSHOT" >&2
|
||||
}
|
||||
|
||||
case "$(cat "$journal/phase")" in
|
||||
committed) start_units && finish committed; exit $? ;;
|
||||
rolled-back) start_units && finish rolled-back; exit $? ;;
|
||||
esac
|
||||
if [ -z "${1:-}" ] && promote; then
|
||||
write_report passed false && finish committed
|
||||
exit $?
|
||||
fi
|
||||
if [ "$(cat "$journal/phase")" = prepared ]; then
|
||||
echo "update was not promoted; leaving running work alone" >&2
|
||||
for cmd in $BINARIES; do rm -f "$BIN_DIR/.$cmd.new"; done
|
||||
write_report failed false && finish rolled-back || exit 1
|
||||
if [ "${1:-}" = --boot-recover ]; then start_units; exit $?; fi
|
||||
exit 1
|
||||
fi
|
||||
echo "==> restoring interrupted or unhealthy update" >&2
|
||||
if rollback; then
|
||||
if [[ ",$COMPONENTS," == *,daemon,* ]]; then explain_database_recovery; fi
|
||||
exit 0
|
||||
fi
|
||||
echo "rollback incomplete; retained journal: $journal" >&2
|
||||
exit 1
|
||||
+1
-2
@@ -94,8 +94,7 @@ archied_dirs() {
|
||||
component_dirs ./cmd/archie-ui
|
||||
printf '%s\n' "cmd/archied" "Dockerfile.archied" "cmd/archie-ui" "internal/app/archieui" \
|
||||
"Taskfile.yml" ".github/workflows/deploy.yml" "install.sh" \
|
||||
"scripts/archie-update-install" \
|
||||
"scripts/archie-update-watchdog" "deployments/INSTRUCTIONS.md"
|
||||
"deployments/INSTRUCTIONS.md"
|
||||
}
|
||||
|
||||
# The UI Service ships with the archied release on purpose (archie-ui shares
|
||||
|
||||
Reference in new issue
Block a user