399 lines
14 KiB
Go
399 lines
14 KiB
Go
package archied
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/BurntSushi/toml"
|
|
"golang.org/x/sys/unix"
|
|
|
|
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
|
"github.com/samcharles93/archie-core/internal/domain/distribution"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
|
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
|
)
|
|
|
|
// The watchdog's patience: how long the restarted daemon has to report the
|
|
// release, and how long it must then stay healthy.
|
|
const (
|
|
updateHealthTimeout = 60 * time.Second
|
|
updateObserve = 120 * time.Second
|
|
updatePoll = 3 * time.Second
|
|
)
|
|
|
|
const updateUsage = `usage:
|
|
archied update check [-channel stable|next|exact-pin] [-pin version]
|
|
archied update install -version X [-config FILE] [-report FILE -channel C -chat ID -thread ID]
|
|
archied update watchdog [-config FILE] [-recover]
|
|
archied update agent-image [-config FILE] REF`
|
|
|
|
// RunUpdate serves `archied update`: check prints the release snapshot,
|
|
// install verifies and stages a release and hands it to the watchdog, and
|
|
// watchdog switches over, watches the daemon and commits or rolls back.
|
|
// agent-image points new tasks at an agent image built outside a release.
|
|
func RunUpdate(args []string, out, stderr io.Writer) int {
|
|
if len(args) == 0 {
|
|
fmt.Fprintln(stderr, updateUsage)
|
|
return 2
|
|
}
|
|
switch args[0] {
|
|
case "check":
|
|
return runUpdateCheck(args[1:], out, stderr)
|
|
case "install":
|
|
return runUpdateInstall(args[1:], out, stderr)
|
|
case "watchdog":
|
|
return runUpdateWatchdog(args[1:], stderr)
|
|
case "agent-image":
|
|
return runUpdateAgentImage(args[1:], out, stderr)
|
|
default:
|
|
fmt.Fprintln(stderr, updateUsage)
|
|
return 2
|
|
}
|
|
}
|
|
|
|
// runUpdateAgentImage sets the image new tasks run. A release install pins
|
|
// its image by digest, so a locally loaded build is otherwise never used.
|
|
func runUpdateAgentImage(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("update agent-image", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
if flags.NArg() != 1 {
|
|
fmt.Fprintln(stderr, updateUsage)
|
|
return 2
|
|
}
|
|
ctx := context.Background()
|
|
policy := imagePolicy{configPath: *configPath}
|
|
current, err := policy.Image(ctx)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
if err := policy.SetImage(ctx, current, flags.Arg(0)); err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
fmt.Fprintf(out, "agent image: %s -> %s\n", current, flags.Arg(0))
|
|
return 0
|
|
}
|
|
|
|
func runUpdateInstall(args []string, out, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("update install", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
|
|
version := flags.String("version", "", "release version to install")
|
|
releases := sourceFlags(flags)
|
|
var meta releaseupdate.InstallMeta
|
|
flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome")
|
|
flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on")
|
|
flags.Int64Var(&meta.ChatID, "chat", 0, "chat the outcome is reported to")
|
|
flags.IntVar(&meta.ThreadID, "thread", 0, "thread the outcome is reported to")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
source, err := releases()
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 2
|
|
}
|
|
self, err := os.Executable()
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
ctx := context.Background()
|
|
tx, err := hostTransaction(*configPath, filepath.Dir(self), func(line string) { fmt.Fprintln(out, line) })
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{
|
|
Version: *version, Source: source, Meta: meta,
|
|
CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error {
|
|
return checkUpdateTopology(ctx, *configPath, required, managed, out)
|
|
},
|
|
})
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
|
|
if err := spawnWatchdog(ctx, result.RestartRequested, frozen, *configPath, false); err != nil {
|
|
// The journal stays: the next archied boot recovers it.
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
if !result.RestartRequested {
|
|
fmt.Fprintf(out, "no systemd user manager: the watchdog installs %s without restarting; restart the archie services to run it\n", *version)
|
|
}
|
|
data, err := json.Marshal(result)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
fmt.Fprintln(out, releaseupdate.ResultSentinel+string(data))
|
|
return 0
|
|
}
|
|
|
|
func runUpdateWatchdog(args []string, stderr io.Writer) int {
|
|
flags := flag.NewFlagSet("update watchdog", flag.ContinueOnError)
|
|
flags.SetOutput(stderr)
|
|
configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file")
|
|
recovering := flags.Bool("recover", false, "recover an interrupted transaction at boot")
|
|
binDir := flags.String("bin-dir", "", "directory of the live binaries")
|
|
if err := flags.Parse(args); err != nil {
|
|
return 2
|
|
}
|
|
tx, err := hostTransaction(*configPath, *binDir, func(line string) { fmt.Fprintln(stderr, line) })
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
if err := tx.Run(context.Background(), *recovering); err != nil {
|
|
fmt.Fprintln(stderr, err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// hostTransaction wires the transaction to this host: systemd user units,
|
|
// Docker, the daemon's health endpoint, the control-plane image policy and
|
|
// the State Store's own backup and restore.
|
|
func hostTransaction(configPath, binDir string, progress func(string)) (*releaseupdate.Transaction, error) {
|
|
_, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if binDir == "" {
|
|
return nil, errors.New("-bin-dir is required")
|
|
}
|
|
return &releaseupdate.Transaction{
|
|
StateDir: updateStateDir(),
|
|
BinDir: binDir,
|
|
Units: systemdUnits{},
|
|
Images: dockerImages{},
|
|
Policy: imagePolicy{configPath: configPath},
|
|
Database: stateDatabase{configPath: configPath},
|
|
Health: daemonHealth{url: doc.Config.Health.URL()},
|
|
Progress: progress,
|
|
HealthTimeout: updateHealthTimeout,
|
|
Observe: updateObserve,
|
|
Poll: updatePoll,
|
|
}, nil
|
|
}
|
|
|
|
// spawnWatchdog runs the frozen watchdog outside this process, so restarting
|
|
// any service, including the one that asked for the update, cannot kill it.
|
|
// Under systemd it is a transient unit of its own; without one it is a
|
|
// detached session.
|
|
func spawnWatchdog(ctx context.Context, managed bool, binary, configPath string, recovering bool) error {
|
|
exe, err := os.Executable()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
args := []string{"update", "watchdog", "-config", configPath, "-bin-dir", filepath.Dir(exe)}
|
|
if recovering {
|
|
args = append(args, "-recover")
|
|
}
|
|
stateEnv := "ARCHIE_UPDATE_STATE_DIR=" + updateStateDir()
|
|
if managed {
|
|
// The units load the env file beside the config for database and
|
|
// service credentials; the watchdog needs the same ones.
|
|
envFile := "EnvironmentFile=-" + filepath.Join(filepath.Dir(configPath), "env")
|
|
run := append([]string{"--user", "--collect", "--quiet", "--unit=" + instanceUnit("archied-update-watchdog"), "--setenv=" + stateEnv, "--setenv=ARCHIE_INSTANCE=" + configuration.InstanceName(), "--property=" + envFile, binary}, args...)
|
|
if output, err := exec.CommandContext(ctx, "systemd-run", run...).CombinedOutput(); err != nil {
|
|
return fmt.Errorf("start the update watchdog: %w: %s", err, output)
|
|
}
|
|
return nil
|
|
}
|
|
logFile, err := os.OpenFile(filepath.Join(updateStateDir(), "watchdog.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = logFile.Close() }()
|
|
cmd := exec.Command(binary, args...) //nolint:noctx // the watchdog must outlive this process
|
|
cmd.Env = append(os.Environ(), stateEnv)
|
|
cmd.Stdout, cmd.Stderr = logFile, logFile
|
|
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
|
if err := cmd.Start(); err != nil {
|
|
return fmt.Errorf("start the update watchdog: %w", err)
|
|
}
|
|
return cmd.Process.Release()
|
|
}
|
|
|
|
// checkUpdateTopology refuses a host missing a unit or config section the
|
|
// release requires. Units are observable only under systemd; without it there
|
|
// is nothing to restart, so only the config is checked.
|
|
func checkUpdateTopology(ctx context.Context, configPath string, required releaseupdate.Topology, managed bool, out io.Writer) error {
|
|
data, err := os.ReadFile(configPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var document map[string]any
|
|
if err := toml.Unmarshal(data, &document); err != nil {
|
|
return fmt.Errorf("observe topology: %w", err)
|
|
}
|
|
sections := make(map[string]bool)
|
|
for _, section := range required.ConfigSections {
|
|
sections[section] = hasConfigSection(document, section)
|
|
}
|
|
// release.json names base units; this instance's units carry its suffix.
|
|
required.Units = slices.Clone(required.Units)
|
|
for i, unit := range required.Units {
|
|
required.Units[i] = instanceUnit(unit)
|
|
}
|
|
units := make(map[string]bool)
|
|
var observeErr error
|
|
if managed {
|
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
var output []byte
|
|
output, observeErr = exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output()
|
|
units = observedUnits(output, observeErr)
|
|
} else {
|
|
for _, unit := range required.Units {
|
|
units[unit] = true
|
|
}
|
|
}
|
|
plan := required.MigrationPlan(units, sections)
|
|
if len(plan) == 0 {
|
|
return nil
|
|
}
|
|
fmt.Fprintln(out, "Required migration, in order:")
|
|
for _, step := range plan {
|
|
fmt.Fprintln(out, " -", step)
|
|
}
|
|
if observeErr != nil {
|
|
return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr)
|
|
}
|
|
return errors.New("refusing partial installation; apply the printed migration before retrying")
|
|
}
|
|
|
|
func hasConfigSection(document map[string]any, section string) bool {
|
|
for part := range strings.SplitSeq(section, ".") {
|
|
next, ok := document[part].(map[string]any)
|
|
if !ok {
|
|
return false
|
|
}
|
|
document = next
|
|
}
|
|
return true
|
|
}
|
|
|
|
func updateStateDir() string {
|
|
if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" {
|
|
return dir
|
|
}
|
|
return configuration.DefaultUpdateStateDir()
|
|
}
|
|
|
|
// resumeUpdate hands an interrupted transaction to the frozen watchdog and
|
|
// reports that this boot must stop: recovery may replace this executable and
|
|
// restart this unit, so it runs outside it.
|
|
func resumeUpdate(ctx context.Context, configPath string) (bool, error) {
|
|
orphaned, err := updateOrphaned()
|
|
if err != nil || !orphaned {
|
|
return false, err
|
|
}
|
|
if err := spawnRecovery(ctx, configPath); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// updateOrphaned reports a journal that no live watchdog holds the lock for.
|
|
func updateOrphaned() (bool, error) {
|
|
journal := releaseupdate.JournalPath(updateStateDir())
|
|
if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) {
|
|
return false, nil
|
|
} else if err != nil {
|
|
return false, err
|
|
}
|
|
lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer lock.Close()
|
|
if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) {
|
|
// A live watchdog owns the transaction and is restarting this unit.
|
|
return false, nil
|
|
} else if err != nil {
|
|
return false, err
|
|
}
|
|
_ = unix.Flock(int(lock.Fd()), unix.LOCK_UN)
|
|
return true, nil
|
|
}
|
|
|
|
func spawnRecovery(ctx context.Context, configPath string) error {
|
|
frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary)
|
|
if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil {
|
|
return fmt.Errorf("schedule update recovery: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// watchUpdateRecovery recovers a transaction whose watchdog died while this
|
|
// daemon kept running. An orphan must be seen on two consecutive checks so a
|
|
// journal published just before its watchdog takes the lock is left alone.
|
|
func watchUpdateRecovery(ctx context.Context, configPath string, every time.Duration) {
|
|
ticker := time.NewTicker(every)
|
|
defer ticker.Stop()
|
|
seen := false
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
orphaned, err := updateOrphaned()
|
|
if err != nil {
|
|
slog.WarnContext(ctx, "check update recovery", "error", err)
|
|
continue
|
|
}
|
|
if orphaned && seen {
|
|
if err := spawnRecovery(ctx, configPath); err != nil {
|
|
slog.WarnContext(ctx, "recover update", "error", err)
|
|
}
|
|
}
|
|
seen = orphaned
|
|
}
|
|
}
|
|
|
|
func observedUnits(output []byte, err error) map[string]bool {
|
|
units := make(map[string]bool)
|
|
if err != nil {
|
|
return units
|
|
}
|
|
for line := range strings.SplitSeq(string(output), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) >= 2 && fields[1] != "masked" {
|
|
units[strings.TrimSuffix(fields[0], ".service")] = true
|
|
}
|
|
}
|
|
return units
|
|
}
|
|
|
|
// sourceFlags registers -source and -key, the release distribution source; a
|
|
// blank pair is the official releases.
|
|
func sourceFlags(flags *flag.FlagSet) func() (distribution.Source, error) {
|
|
reference := flags.String("source", "", "OCI repository releases are published to (default: the official releases)")
|
|
key := flags.String("key", "", "base64 ed25519 public key the source signs releases with")
|
|
return func() (distribution.Source, error) {
|
|
return distribution.Parse(*reference, *key, distribution.DefaultReleases())
|
|
}
|
|
}
|