Files
archie/internal/app/archied/update.go
T
sam 9d8f364af1
Quality / check (push) Successful in 5m53s
deploy / gate (push) Canceled after 0s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
feat(releaseupdate): signed OCI distribution sources for releases and the catalogue (#1715)
2026-10-31 18:18:43 +11:00

399 lines
14 KiB
Go

package archied
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log/slog"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"syscall"
"time"
"github.com/BurntSushi/toml"
"golang.org/x/sys/unix"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/distribution"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/releaseupdate"
)
// The watchdog's patience: how long the restarted daemon has to report the
// release, and how long it must then stay healthy.
const (
updateHealthTimeout = 60 * time.Second
updateObserve = 120 * time.Second
updatePoll = 3 * time.Second
)
const updateUsage = `usage:
archied update check [-channel stable|next|exact-pin] [-pin version]
archied update install -version X [-config FILE] [-report FILE -channel C -chat ID -thread ID]
archied update watchdog [-config FILE] [-recover]
archied update agent-image [-config FILE] REF`
// RunUpdate serves `archied update`: check prints the release snapshot,
// install verifies and stages a release and hands it to the watchdog, and
// watchdog switches over, watches the daemon and commits or rolls back.
// agent-image points new tasks at an agent image built outside a release.
func RunUpdate(args []string, out, stderr io.Writer) int {
if len(args) == 0 {
fmt.Fprintln(stderr, updateUsage)
return 2
}
switch args[0] {
case "check":
return runUpdateCheck(args[1:], out, stderr)
case "install":
return runUpdateInstall(args[1:], out, stderr)
case "watchdog":
return runUpdateWatchdog(args[1:], stderr)
case "agent-image":
return runUpdateAgentImage(args[1:], out, stderr)
default:
fmt.Fprintln(stderr, updateUsage)
return 2
}
}
// runUpdateAgentImage sets the image new tasks run. A release install pins
// its image by digest, so a locally loaded build is otherwise never used.
func runUpdateAgentImage(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("update agent-image", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
if err := flags.Parse(args); err != nil {
return 2
}
if flags.NArg() != 1 {
fmt.Fprintln(stderr, updateUsage)
return 2
}
ctx := context.Background()
policy := imagePolicy{configPath: *configPath}
current, err := policy.Image(ctx)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err := policy.SetImage(ctx, current, flags.Arg(0)); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintf(out, "agent image: %s -> %s\n", current, flags.Arg(0))
return 0
}
func runUpdateInstall(args []string, out, stderr io.Writer) int {
flags := flag.NewFlagSet("update install", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
version := flags.String("version", "", "release version to install")
releases := sourceFlags(flags)
var meta releaseupdate.InstallMeta
flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome")
flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on")
flags.Int64Var(&meta.ChatID, "chat", 0, "chat the outcome is reported to")
flags.IntVar(&meta.ThreadID, "thread", 0, "thread the outcome is reported to")
if err := flags.Parse(args); err != nil {
return 2
}
source, err := releases()
if err != nil {
fmt.Fprintln(stderr, err)
return 2
}
self, err := os.Executable()
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
ctx := context.Background()
tx, err := hostTransaction(*configPath, filepath.Dir(self), func(line string) { fmt.Fprintln(out, line) })
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
Version: *version, Source: source, Meta: meta,
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
return checkUpdateTopology(ctx, *configPath, required, managed, out)
},
})
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
if err := spawnWatchdog(ctx, result.RestartRequested, frozen, *configPath, false); err != nil {
// The journal stays: the next archied boot recovers it.
fmt.Fprintln(stderr, err)
return 1
}
if !result.RestartRequested {
fmt.Fprintf(out, "no systemd user manager: the watchdog installs %s without restarting; restart the archie services to run it\n", *version)
}
data, err := json.Marshal(result)
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
fmt.Fprintln(out, releaseupdate.ResultSentinel+string(data))
return 0
}
func runUpdateWatchdog(args []string, stderr io.Writer) int {
flags := flag.NewFlagSet("update watchdog", flag.ContinueOnError)
flags.SetOutput(stderr)
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
recovering := flags.Bool("recover", false, "recover an interrupted transaction at boot")
binDir := flags.String("bin-dir", "", "directory of the live binaries")
if err := flags.Parse(args); err != nil {
return 2
}
tx, err := hostTransaction(*configPath, *binDir, func(line string) { fmt.Fprintln(stderr, line) })
if err != nil {
fmt.Fprintln(stderr, err)
return 1
}
if err := tx.Run(context.Background(), *recovering); err != nil {
fmt.Fprintln(stderr, err)
return 1
}
return 0
}
// hostTransaction wires the transaction to this host: systemd user units,
// Docker, the daemon's health endpoint, the control-plane image policy and
// the State Store's own backup and restore.
func hostTransaction(configPath, binDir string, progress func(string)) (*releaseupdate.Transaction, error) {
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
if err != nil {
return nil, err
}
if binDir == "" {
return nil, errors.New("-bin-dir is required")
}
return &releaseupdate.Transaction{
StateDir: updateStateDir(),
BinDir: binDir,
Units: systemdUnits{},
Images: dockerImages{},
Policy: imagePolicy{configPath: configPath},
Database: stateDatabase{configPath: configPath},
Health: daemonHealth{url: doc.Config.Health.URL()},
Progress: progress,
HealthTimeout: updateHealthTimeout,
Observe: updateObserve,
Poll: updatePoll,
}, nil
}
// spawnWatchdog runs the frozen watchdog outside this process, so restarting
// any service, including the one that asked for the update, cannot kill it.
// Under systemd it is a transient unit of its own; without one it is a
// detached session.
func spawnWatchdog(ctx context.Context, managed bool, binary, configPath string, recovering bool) error {
exe, err := os.Executable()
if err != nil {
return err
}
args := []string{"update", "watchdog", "-config", configPath, "-bin-dir", filepath.Dir(exe)}
if recovering {
args = append(args, "-recover")
}
stateEnv := "ARCHIE_UPDATE_STATE_DIR=" + updateStateDir()
if managed {
// The units load the env file beside the config for database and
// service credentials; the watchdog needs the same ones.
envFile := "EnvironmentFile=-" + filepath.Join(filepath.Dir(configPath), "env")
run := append([]string{"--user", "--collect", "--quiet", "--unit=" + instanceUnit("archied-update-watchdog"), "--setenv=" + stateEnv, "--setenv=ARCHIE_INSTANCE=" + configuration.InstanceName(), "--property=" + envFile, binary}, args...)
if output, err := exec.CommandContext(ctx, "systemd-run", run...).CombinedOutput(); err != nil {
return fmt.Errorf("start the update watchdog: %w: %s", err, output)
}
return nil
}
logFile, err := os.OpenFile(filepath.Join(updateStateDir(), "watchdog.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer func() { _ = logFile.Close() }()
cmd := exec.Command(binary, args...) //nolint:noctx // the watchdog must outlive this process
cmd.Env = append(os.Environ(), stateEnv)
cmd.Stdout, cmd.Stderr = logFile, logFile
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
if err := cmd.Start(); err != nil {
return fmt.Errorf("start the update watchdog: %w", err)
}
return cmd.Process.Release()
}
// checkUpdateTopology refuses a host missing a unit or config section the
// release requires. Units are observable only under systemd; without it there
// is nothing to restart, so only the config is checked.
func checkUpdateTopology(ctx context.Context, configPath string, required releaseupdate.Topology, managed bool, out io.Writer) error {
data, err := os.ReadFile(configPath)
if err != nil {
return err
}
var document map[string]any
if err := toml.Unmarshal(data, &document); err != nil {
return fmt.Errorf("observe topology: %w", err)
}
sections := make(map[string]bool)
for _, section := range required.ConfigSections {
sections[section] = hasConfigSection(document, section)
}
// release.json names base units; this instance's units carry its suffix.
required.Units = slices.Clone(required.Units)
for i, unit := range required.Units {
required.Units[i] = instanceUnit(unit)
}
units := make(map[string]bool)
var observeErr error
if managed {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
var output []byte
output, observeErr = exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
units = observedUnits(output, observeErr)
} else {
for _, unit := range required.Units {
units[unit] = true
}
}
plan := required.MigrationPlan(units, sections)
if len(plan) == 0 {
return nil
}
fmt.Fprintln(out, "Required migration, in order:")
for _, step := range plan {
fmt.Fprintln(out, " -", step)
}
if observeErr != nil {
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
}
return errors.New("refusing partial installation; apply the printed migration before retrying")
}
func hasConfigSection(document map[string]any, section string) bool {
for part := range strings.SplitSeq(section, ".") {
next, ok := document[part].(map[string]any)
if !ok {
return false
}
document = next
}
return true
}
func updateStateDir() string {
if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" {
return dir
}
return configuration.DefaultUpdateStateDir()
}
// resumeUpdate hands an interrupted transaction to the frozen watchdog and
// reports that this boot must stop: recovery may replace this executable and
// restart this unit, so it runs outside it.
func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
orphaned, err := updateOrphaned()
if err != nil || !orphaned {
return false, err
}
if err := spawnRecovery(ctx, configPath); err != nil {
return false, err
}
return true, nil
}
// updateOrphaned reports a journal that no live watchdog holds the lock for.
func updateOrphaned() (bool, error) {
journal := releaseupdate.JournalPath(updateStateDir())
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
return false, nil
} else if err != nil {
return false, err
}
lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return false, err
}
defer lock.Close()
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) {
// A live watchdog owns the transaction and is restarting this unit.
return false, nil
} else if err != nil {
return false, err
}
_ = unix.Flock(int(lock.Fd()), unix.LOCK_UN)
return true, nil
}
func spawnRecovery(ctx context.Context, configPath string) error {
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
return fmt.Errorf("schedule update recovery: %w", err)
}
return nil
}
// watchUpdateRecovery recovers a transaction whose watchdog died while this
// daemon kept running. An orphan must be seen on two consecutive checks so a
// journal published just before its watchdog takes the lock is left alone.
func watchUpdateRecovery(ctx context.Context, configPath string, every time.Duration) {
ticker := time.NewTicker(every)
defer ticker.Stop()
seen := false
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
orphaned, err := updateOrphaned()
if err != nil {
slog.WarnContext(ctx, "check update recovery", "error", err)
continue
}
if orphaned && seen {
if err := spawnRecovery(ctx, configPath); err != nil {
slog.WarnContext(ctx, "recover update", "error", err)
}
}
seen = orphaned
}
}
func observedUnits(output []byte, err error) map[string]bool {
units := make(map[string]bool)
if err != nil {
return units
}
for line := range strings.SplitSeq(string(output), "\n") {
fields := strings.Fields(line)
if len(fields) >= 2 && fields[1] != "masked" {
units[strings.TrimSuffix(fields[0], ".service")] = true
}
}
return units
}
// sourceFlags registers -source and -key, the release distribution source; a
// blank pair is the official releases.
func sourceFlags(flags *flag.FlagSet) func() (distribution.Source, error) {
reference := flags.String("source", "", "OCI repository releases are published to (default: the official releases)")
key := flags.String("key", "", "base64 ed25519 public key the source signs releases with")
return func() (distribution.Source, error) {
return distribution.Parse(*reference, *key, distribution.DefaultReleases())
}
}