package archied import ( "context" "encoding/json" "errors" "flag" "fmt" "io" "log/slog" "os" "os/exec" "path/filepath" "slices" "strings" "syscall" "time" "github.com/BurntSushi/toml" "golang.org/x/sys/unix" "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/releaseupdate" ) // The watchdog's patience: how long the restarted daemon has to report the // release, and how long it must then stay healthy. const ( updateHealthTimeout = 60 * time.Second updateObserve = 120 * time.Second updatePoll = 3 * time.Second ) const updateUsage = `usage: archied update check [-channel stable|next|exact-pin] [-pin version] archied update install -version X [-config FILE] [-report FILE -channel C -chat ID -thread ID] archied update watchdog [-config FILE] [-recover] archied update agent-image [-config FILE] REF` // RunUpdate serves `archied update`: check prints the release snapshot, // install verifies and stages a release and hands it to the watchdog, and // watchdog switches over, watches the daemon and commits or rolls back. // agent-image points new tasks at an agent image built outside a release. func RunUpdate(args []string, out, stderr io.Writer) int { if len(args) == 0 { fmt.Fprintln(stderr, updateUsage) return 2 } switch args[0] { case "check": return runUpdateCheck(args[1:], out, stderr) case "install": return runUpdateInstall(args[1:], out, stderr) case "watchdog": return runUpdateWatchdog(args[1:], stderr) case "agent-image": return runUpdateAgentImage(args[1:], out, stderr) default: fmt.Fprintln(stderr, updateUsage) return 2 } } // runUpdateAgentImage sets the image new tasks run. A release install pins // its image by digest, so a locally loaded build is otherwise never used. func runUpdateAgentImage(args []string, out, stderr io.Writer) int { flags := flag.NewFlagSet("update agent-image", flag.ContinueOnError) flags.SetOutput(stderr) configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") if err := flags.Parse(args); err != nil { return 2 } if flags.NArg() != 1 { fmt.Fprintln(stderr, updateUsage) return 2 } ctx := context.Background() policy := imagePolicy{configPath: *configPath} current, err := policy.Image(ctx) if err != nil { fmt.Fprintln(stderr, err) return 1 } if err := policy.SetImage(ctx, current, flags.Arg(0)); err != nil { fmt.Fprintln(stderr, err) return 1 } fmt.Fprintf(out, "agent image: %s -> %s\n", current, flags.Arg(0)) return 0 } func runUpdateInstall(args []string, out, stderr io.Writer) int { flags := flag.NewFlagSet("update install", flag.ContinueOnError) flags.SetOutput(stderr) configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") version := flags.String("version", "", "release version to install") releaseURL := flags.String("release-url", releaseupdate.ReleaseURL, "base URL of the release downloads") var meta releaseupdate.InstallMeta flags.StringVar(&meta.ReportPath, "report", "", "where the watchdog leaves the outcome") flags.StringVar(&meta.Channel, "channel", "", "channel the outcome is reported on") flags.Int64Var(&meta.ChatID, "chat", 0, "chat the outcome is reported to") flags.IntVar(&meta.ThreadID, "thread", 0, "thread the outcome is reported to") if err := flags.Parse(args); err != nil { return 2 } self, err := os.Executable() if err != nil { fmt.Fprintln(stderr, err) return 1 } ctx := context.Background() tx, err := hostTransaction(*configPath, filepath.Dir(self), func(line string) { fmt.Fprintln(out, line) }) if err != nil { fmt.Fprintln(stderr, err) return 1 } result, err := tx.Prepare(ctx, releaseupdate.PrepareRequest{ Version: *version, ReleaseURL: *releaseURL, Meta: meta, CheckTopology: func(ctx context.Context, required releaseupdate.Topology, managed bool) error { return checkUpdateTopology(ctx, *configPath, required, managed, out) }, }) if err != nil { fmt.Fprintln(stderr, err) return 1 } frozen := filepath.Join(releaseupdate.JournalPath(updateStateDir()), releaseupdate.WatchdogBinary) if err := spawnWatchdog(ctx, result.RestartRequested, frozen, *configPath, false); err != nil { // The journal stays: the next archied boot recovers it. fmt.Fprintln(stderr, err) return 1 } if !result.RestartRequested { fmt.Fprintf(out, "no systemd user manager: the watchdog installs %s without restarting; restart the archie services to run it\n", *version) } data, err := json.Marshal(result) if err != nil { fmt.Fprintln(stderr, err) return 1 } fmt.Fprintln(out, releaseupdate.ResultSentinel+string(data)) return 0 } func runUpdateWatchdog(args []string, stderr io.Writer) int { flags := flag.NewFlagSet("update watchdog", flag.ContinueOnError) flags.SetOutput(stderr) configPath := flags.String("config", configuration.DefaultConfigPath(), "bootstrap config file") recovering := flags.Bool("recover", false, "recover an interrupted transaction at boot") binDir := flags.String("bin-dir", "", "directory of the live binaries") if err := flags.Parse(args); err != nil { return 2 } tx, err := hostTransaction(*configPath, *binDir, func(line string) { fmt.Fprintln(stderr, line) }) if err != nil { fmt.Fprintln(stderr, err) return 1 } if err := tx.Run(context.Background(), *recovering); err != nil { fmt.Fprintln(stderr, err) return 1 } return 0 } // hostTransaction wires the transaction to this host: systemd user units, // Docker, the daemon's health endpoint, the control-plane image policy and // the State Store's own backup and restore. func hostTransaction(configPath, binDir string, progress func(string)) (*releaseupdate.Transaction, error) { _, doc, err := servicekit.Resolve(slog.New(slog.DiscardHandler), configPath, "") if err != nil { return nil, err } if binDir == "" { return nil, errors.New("-bin-dir is required") } return &releaseupdate.Transaction{ StateDir: updateStateDir(), BinDir: binDir, Units: systemdUnits{}, Images: dockerImages{}, Policy: imagePolicy{configPath: configPath}, Database: stateDatabase{configPath: configPath}, Health: daemonHealth{url: doc.Config.Health.URL()}, Progress: progress, HealthTimeout: updateHealthTimeout, Observe: updateObserve, Poll: updatePoll, }, nil } // spawnWatchdog runs the frozen watchdog outside this process, so restarting // any service, including the one that asked for the update, cannot kill it. // Under systemd it is a transient unit of its own; without one it is a // detached session. func spawnWatchdog(ctx context.Context, managed bool, binary, configPath string, recovering bool) error { exe, err := os.Executable() if err != nil { return err } args := []string{"update", "watchdog", "-config", configPath, "-bin-dir", filepath.Dir(exe)} if recovering { args = append(args, "-recover") } stateEnv := "ARCHIE_UPDATE_STATE_DIR=" + updateStateDir() if managed { // The units load the env file beside the config for database and // service credentials; the watchdog needs the same ones. envFile := "EnvironmentFile=-" + filepath.Join(filepath.Dir(configPath), "env") run := append([]string{"--user", "--collect", "--quiet", "--unit=" + instanceUnit("archied-update-watchdog"), "--setenv=" + stateEnv, "--setenv=ARCHIE_INSTANCE=" + configuration.InstanceName(), "--property=" + envFile, binary}, args...) if output, err := exec.CommandContext(ctx, "systemd-run", run...).CombinedOutput(); err != nil { return fmt.Errorf("start the update watchdog: %w: %s", err, output) } return nil } logFile, err := os.OpenFile(filepath.Join(updateStateDir(), "watchdog.log"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) if err != nil { return err } defer func() { _ = logFile.Close() }() cmd := exec.Command(binary, args...) //nolint:noctx // the watchdog must outlive this process cmd.Env = append(os.Environ(), stateEnv) cmd.Stdout, cmd.Stderr = logFile, logFile cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} if err := cmd.Start(); err != nil { return fmt.Errorf("start the update watchdog: %w", err) } return cmd.Process.Release() } // checkUpdateTopology refuses a host missing a unit or config section the // release requires. Units are observable only under systemd; without it there // is nothing to restart, so only the config is checked. func checkUpdateTopology(ctx context.Context, configPath string, required releaseupdate.Topology, managed bool, out io.Writer) error { data, err := os.ReadFile(configPath) if err != nil { return err } var document map[string]any if err := toml.Unmarshal(data, &document); err != nil { return fmt.Errorf("observe topology: %w", err) } sections := make(map[string]bool) for _, section := range required.ConfigSections { sections[section] = hasConfigSection(document, section) } // release.json names base units; this instance's units carry its suffix. required.Units = slices.Clone(required.Units) for i, unit := range required.Units { required.Units[i] = instanceUnit(unit) } units := make(map[string]bool) var observeErr error if managed { ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() var output []byte output, observeErr = exec.CommandContext(ctx, "systemctl", "--user", "list-unit-files", "archie*.service", "--no-legend", "--no-pager").Output() units = observedUnits(output, observeErr) } else { for _, unit := range required.Units { units[unit] = true } } plan := required.MigrationPlan(units, sections) if len(plan) == 0 { return nil } fmt.Fprintln(out, "Required migration, in order:") for _, step := range plan { fmt.Fprintln(out, " -", step) } if observeErr != nil { return fmt.Errorf("topology unknown; apply the printed migration before retrying: %w", observeErr) } return errors.New("refusing partial installation; apply the printed migration before retrying") } func hasConfigSection(document map[string]any, section string) bool { for part := range strings.SplitSeq(section, ".") { next, ok := document[part].(map[string]any) if !ok { return false } document = next } return true } func updateStateDir() string { if dir := os.Getenv("ARCHIE_UPDATE_STATE_DIR"); dir != "" { return dir } return configuration.DefaultUpdateStateDir() } // resumeUpdate hands an interrupted transaction to the frozen watchdog and // reports that this boot must stop: recovery may replace this executable and // restart this unit, so it runs outside it. func resumeUpdate(ctx context.Context, configPath string) (bool, error) { journal := releaseupdate.JournalPath(updateStateDir()) if _, err := os.Stat(journal); errors.Is(err, os.ErrNotExist) { return false, nil } else if err != nil { return false, err } lock, err := os.OpenFile(filepath.Join(updateStateDir(), "lock"), os.O_CREATE|os.O_RDWR, 0o600) if err != nil { return false, err } defer lock.Close() if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX|unix.LOCK_NB); errors.Is(err, unix.EWOULDBLOCK) { // A live watchdog owns the transaction and is restarting this unit. return false, nil } else if err != nil { return false, err } _ = unix.Flock(int(lock.Fd()), unix.LOCK_UN) frozen := filepath.Join(journal, releaseupdate.WatchdogBinary) if err := spawnWatchdog(ctx, (systemdUnits{}).Managed(ctx), frozen, configPath, true); err != nil { return false, fmt.Errorf("schedule update recovery: %w", err) } return true, nil } func observedUnits(output []byte, err error) map[string]bool { units := make(map[string]bool) if err != nil { return units } for line := range strings.SplitSeq(string(output), "\n") { fields := strings.Fields(line) if len(fields) >= 2 && fields[1] != "masked" { units[strings.TrimSuffix(fields[0], ".service")] = true } } return units }