diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 09c7fa870..0f7b38e28 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -1566,6 +1566,13 @@ func (b *boot) runLoop(ctx context.Context, once bool) error { // Boot is done: the liveness surface stops answering 503, which is what // the update watchdog is waiting to see after a restart. b.health.markServing() + // systemd's READY=1 asserts the same fact this line has just recorded, so + // it is sent here rather than from a second notion of "started". Not + // earlier: the stores, NATS, gateways and tool providers are all up by now. + // Not later, after the first pass: a pass polls every repo and drains the + // queue, and a unit whose TimeoutStartSec expires mid-pass would be + // restarted while the daemon was working. + b.announceReady() if once { b.d.Cycle(ctx) return nil @@ -1575,6 +1582,9 @@ func (b *boot) runLoop(ctx context.Context, once bool) error { // a --once cycle exits immediately, so a drain that arrives mid-cycle is // out of scope and the monitor is not started. b.startDrainMonitor(ctx) + // Armed here, not at boot: the loop it reports on starts on the next line, + // and a --once invocation has no loop to watchdog. + b.startWatchdog(ctx) if err := b.d.Run(ctx); err != nil && ctx.Err() == nil { b.log.Error("daemon exited", "err", err) return err diff --git a/internal/app/archied/sd_notify.go b/internal/app/archied/sd_notify.go new file mode 100644 index 000000000..1ca5f8154 --- /dev/null +++ b/internal/app/archied/sd_notify.go @@ -0,0 +1,237 @@ +// sd_notify.go implements systemd's sd_notify(3) protocol for archied: one +// READY=1 when the run loop starts serving, and a WATCHDOG=1 heartbeat while +// the run loop is demonstrably still making progress. +// +// The protocol is three environment variables and one datagram per state, so +// this file speaks it directly instead of taking a module dependency: the whole +// client is net.DialUnix plus a Write. Nothing here is fatal -- a process with +// no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without WatchdogSec) +// sends nothing and serves exactly as before, and a socket that cannot be +// written to is logged and ignored. +// +// The heartbeat is deliberately not a bare ticker. A ticker fires whether or +// not the loop it stands for is alive, so it would keep systemd satisfied while +// the event loop was wedged -- the exact failure the watchdog exists to catch. +// The sampler instead reads the daemon's own progress marker +// (daemon.LastPollAt, stamped by the poll loop at the start of every pass) and +// withholds the heartbeat once that marker is older than the loop's own poll +// cadence. While passes keep beginning, beats flow at half of WatchdogSec; a +// loop blocked inside a pass stops being reported as alive. +// +// Consuming the states is the unit's business: READY=1 needs Type=notify, and +// the heartbeat needs WatchdogSec. +package archied + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "strconv" + "time" +) + +const ( + // notifySocketEnv names the socket a manager passes to a Type=notify unit. + notifySocketEnv = "NOTIFY_SOCKET" + // watchdogUsecEnv carries WatchdogSec in microseconds. + watchdogUsecEnv = "WATCHDOG_USEC" + // watchdogPIDEnv names the process whose beats the watchdog counts. + watchdogPIDEnv = "WATCHDOG_PID" + + // readyState and watchdogState are the two sd_notify states archied sends. + readyState = "READY=1" + watchdogState = "WATCHDOG=1" + + // watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half + // of WatchdogSec, so one missed beat is tolerated and two are not. + watchdogHeartbeatDivisor = 2 +) + +// notifier writes sd_notify states to the socket named in the environment. A +// notifier with no address -- the normal case away from systemd -- sends +// nothing and says nothing. +type notifier struct { + addr string + log *slog.Logger +} + +// newNotifier resolves the notify socket. The address is read from the +// environment systemd sets for the process, so systemd replaces the previous +// unit's socket with the new one across a restart. +func newNotifier(getenv func(string) string, log *slog.Logger) notifier { + return notifier{addr: getenv(notifySocketEnv), log: log} +} + +// send delivers one state line. A failure is logged, never returned: a +// notification is an observation about the daemon, and no observation is worth +// refusing to serve over. +func (n notifier) send(state string) { + if n.addr == "" { + return + } + if err := sendNotify(n.addr, state); err != nil { + n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err) + } +} + +// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is +// one datagram per state, so each send dials afresh: a manager restart recreates +// its socket, and a connection cached across one would fail every send after it. +// A leading '@' names the Linux abstract namespace, which user managers use; +// net.UnixAddr takes that syntax verbatim. +func sendNotify(addr, state string) error { + conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"}) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + if _, err := conn.Write([]byte(state)); err != nil { + return fmt.Errorf("write %q: %w", state, err) + } + return nil +} + +// announceReady tells systemd the daemon has finished starting. It is called at +// the moment the process declares itself serving (runLoop, next to +// healthSurface.markServing), which is the same fact READY=1 asserts: boot is +// over and the run loop is about to take over. +func (b *boot) announceReady() { + newNotifier(os.Getenv, b.log).send(readyState) +} + +// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does +// nothing unless a manager named a notify socket and asked for a watchdog +// (WATCHDOG_USEC), so every deployment without systemd is unaffected. +func (b *boot) startWatchdog(ctx context.Context) { + n := newNotifier(os.Getenv, b.log) + if n.addr == "" { + return + } + interval, ok := watchdogHeartbeat(os.Getenv, os.Getpid(), b.log) + if !ok { + return + } + + sampler := loopWatchdog{ + interval: interval, + cadence: b.loopCadence, + progress: b.d.LastPollAt, + beat: func() { n.send(watchdogState) }, + log: b.log, + } + b.log.Info("systemd watchdog armed", + "heartbeat", interval.String(), + "loop_cadence", b.loopCadence().String()) + + go func() { + if err := sampler.run(ctx); err != nil && !errors.Is(err, context.Canceled) { + b.log.Error("systemd watchdog stopped", "err", err) + } + }() +} + +// loopCadence is the longest interval at which the run loop may go without +// beginning a fresh poll pass, and therefore how stale the progress marker may +// be before the loop counts as stalled. +// +// It is the root poll interval, or the slowest configured identity in +// multi-identity mode: each identity polls its own repos, identity poll +// intervals are frozen at boot, and the marker is stamped by whichever identity +// polled most recently -- so the worst case between stamps is the slowest +// identity, not the fastest. The root value is re-read per sample, so a +// reloaded poll_interval is honoured. +func (b *boot) loopCadence() time.Duration { + cadence := b.cfgHolder.Get().PollInterval.Std() + for _, id := range b.d.Identities { + if override := id.Cfg.PollInterval.Std(); override > cadence { + cadence = override + } + } + return cadence +} + +// watchdogHeartbeat reads systemd's watchdog contract and returns the heartbeat +// cadence, which is half of WatchdogSec. ok is false when systemd asked for no +// watchdog, and also when it asked for one from a different process: +// WATCHDOG_PID names the process whose beats count, so a unit watching another +// process must not have this one beating on its behalf. +func watchdogHeartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) { + raw := getenv(watchdogUsecEnv) + if raw == "" { + return 0, false + } + if rawPID := getenv(watchdogPIDEnv); rawPID != "" { + wantPID, err := strconv.Atoi(rawPID) + if err != nil || wantPID != pid { + log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process", + "watchdog_pid", rawPID, "pid", pid, "err", err) + return 0, false + } + } + watchdog, err := time.ParseDuration(raw + "us") + if err != nil || watchdog <= 0 { + log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err) + return 0, false + } + return watchdog / watchdogHeartbeatDivisor, true +} + +// loopWatchdog samples the run loop's progress marker and emits the watchdog +// heartbeat only while that marker is fresh. Sampling is periodic; the beat is +// not, because it is licensed by the marker rather than by the clock. +type loopWatchdog struct { + // interval is the sampling cadence: half of systemd's WatchdogSec. + interval time.Duration + // cadence reports how long the loop may go between poll passes. + cadence func() time.Duration + // progress reports the start of the loop's most recent poll pass, or the + // zero time when no pass has begun yet. + progress func() time.Time + // beat emits one WATCHDOG=1. + beat func() + log *slog.Logger +} + +// run samples until ctx ends. It returns ctx.Err() on shutdown, which is the +// only way out: this sampler never gives up on its own, because a sampler that +// stopped would leave systemd watching a process nothing reports for. +func (w loopWatchdog) run(ctx context.Context) error { + armedAt := time.Now() + stalled := false + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + + last := w.progress() + if last.IsZero() { + // No pass has begun at all. Measure staleness from arming, so a + // loop that never starts polling is caught like one that stopped. + last = armedAt + } + cadence := w.cadence() + switch age := time.Since(last); { + case age > cadence: + if !stalled { + w.log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does", + "stale_for", age.String(), "loop_cadence", cadence.String()) + stalled = true + } + default: + if stalled { + w.log.Info("systemd watchdog: run loop resumed; heartbeats resumed", + "last_pass_age", age.String()) + stalled = false + } + w.beat() + } + } +} diff --git a/internal/app/archied/sd_notify_test.go b/internal/app/archied/sd_notify_test.go new file mode 100644 index 000000000..7a3f3177e --- /dev/null +++ b/internal/app/archied/sd_notify_test.go @@ -0,0 +1,368 @@ +package archied + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "path/filepath" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/daemon" + "github.com/samcharles93/archie-core/internal/store" +) + +// notifyRecorder captures every record at every level. The other recording +// handler in this package keeps warnings only; these tests need "this path +// logged nothing at all" as positive evidence that a disabled notifier stayed +// silent, so the level has to be part of what is captured. +type notifyRecorder struct { + mu sync.Mutex + records []slog.Record +} + +func (h *notifyRecorder) Enabled(context.Context, slog.Level) bool { return true } + +func (h *notifyRecorder) Handle(_ context.Context, r slog.Record) error { + // The Record is the caller's to reuse once Handle returns, so keep a copy. + rec := r.Clone() + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, rec) + return nil +} + +func (h *notifyRecorder) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *notifyRecorder) WithGroup(string) slog.Handler { return h } + +func (h *notifyRecorder) all() []slog.Record { + h.mu.Lock() + defer h.mu.Unlock() + return append([]slog.Record(nil), h.records...) +} + +func (h *notifyRecorder) at(level slog.Level) []slog.Record { + h.mu.Lock() + defer h.mu.Unlock() + var out []slog.Record + for _, r := range h.records { + if r.Level == level { + out = append(out, r) + } + } + return out +} + +// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket +// that collects the state lines the daemon sends. It is bound to the real +// NOTIFY_SOCKET for the duration of the test. +type notifyListener struct { + conn *net.UnixConn +} + +func newNotifyListener(t *testing.T) *notifyListener { + t.Helper() + // A unix socket path is capped near 108 bytes; the short file name keeps the + // path inside t.TempDir() under that for any test name. + return listenNotify(t, filepath.Join(t.TempDir(), "n.sock")) +} + +// listenNotify binds a notify socket at addr and points NOTIFY_SOCKET at it. +func listenNotify(t *testing.T, addr string) *notifyListener { + t.Helper() + conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"}) + if err != nil { + t.Fatalf("listen for notify datagrams at %q: %v", addr, err) + } + t.Cleanup(func() { _ = conn.Close() }) + t.Setenv(notifySocketEnv, addr) + return ¬ifyListener{conn: conn} +} + +// states drains every state line received within window. +func (l *notifyListener) states(t *testing.T, window time.Duration) []string { + t.Helper() + deadline := time.Now().Add(window) + var got []string + buf := make([]byte, 1024) + for { + if err := l.conn.SetReadDeadline(deadline); err != nil { + t.Fatalf("set notify read deadline: %v", err) + } + n, _, err := l.conn.ReadFromUnix(buf) + if err != nil { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return got + } + t.Fatalf("read notify datagram: %v", err) + } + got = append(got, string(buf[:n])) + } +} + +// countState counts the datagrams carrying want among the received states. +func countState(states []string, want string) int { + n := 0 + for _, state := range states { + if state == want { + n++ + } + } + return n +} + +// newNotifyTestBoot builds the composition the notify path reads: a boot whose +// config holder supplies the run loop's poll cadence, and a real daemon whose +// own poll loop stamps the progress marker the heartbeat sampler samples. +func newNotifyTestBoot(t *testing.T, cadence time.Duration) (*boot, *daemon.Daemon) { + t.Helper() + cfg := config.Config{PollInterval: config.Duration(cadence)} + holder := config.NewHolder(cfg) + st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "tasks.db")) + if err != nil { + t.Fatalf("open task store: %v", err) + } + log := slog.New(slog.DiscardHandler) + d := &daemon.Daemon{Cfg: holder, Store: st, Log: log} + return &boot{cfg: cfg, cfgHolder: holder, d: d, log: log}, d +} + +// pollPasses drives the daemon's poll passes the way Run does, one Cycle per +// tick, while on is set. The progress marker is stamped by the daemon inside +// Cycle, never by the test: what the sampler samples is the loop's own +// evidence, not a value the test wrote. +func pollPasses(ctx context.Context, d *daemon.Daemon, on *atomic.Bool, every time.Duration) { + ticker := time.NewTicker(every) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if on.Load() { + d.Cycle(ctx) + } + } + } +} + +// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not +// restricted to filesystem paths, and a client that stripped the prefix would +// connect to a file that does not exist instead of the abstract namespace. +func TestNotifyReachesAbstractSocket(t *testing.T) { + listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid())) + + (&boot{log: slog.New(slog.DiscardHandler)}).announceReady() + + if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, readyState) { + t.Errorf("states received = %v, want %q among them", states, readyState) + } +} + +// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the +// delivery contract: no socket means nothing is attempted and nothing is said, +// while a socket that cannot be written to is reported once and then ignored. +// Neither may ever be fatal. +func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) { + rec := ¬ifyRecorder{} + log := slog.New(rec) + + newNotifier(func(string) string { return "" }, log).send(readyState) + if got := rec.all(); len(got) != 0 { + t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got) + } + + missing := filepath.Join(t.TempDir(), "absent", "n.sock") + newNotifier(func(string) string { return missing }, log).send(readyState) + warns := rec.at(slog.LevelWarn) + if len(warns) != 1 { + t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns)) + } + if errs := rec.at(slog.LevelError); len(errs) != 0 { + t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs)) + } +} + +// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes +// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and +// WATCHDOG_PID names the process whose beats count. +func TestWatchdogHeartbeatInterval(t *testing.T) { + const self = 42 + tests := []struct { + name string + usec string + pidEnv string + want time.Duration + wantOK bool + }{ + {name: "no watchdog requested", wantOK: false}, + {name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true}, + {name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true}, + {name: "unparseable", usec: "soon", wantOK: false}, + {name: "zero", usec: "0", wantOK: false}, + {name: "negative", usec: "-1", wantOK: false}, + {name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true}, + {name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false}, + {name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + getenv := func(key string) string { + switch key { + case watchdogUsecEnv: + return tt.usec + case watchdogPIDEnv: + return tt.pidEnv + default: + return "" + } + } + got, ok := watchdogHeartbeat(getenv, self, slog.New(slog.DiscardHandler)) + if ok != tt.wantOK || got != tt.want { + t.Errorf("watchdogHeartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v", + tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK) + } + }) + } +} + +// TestNotifyReadyAndWatchdogHeartbeat is the run-loop path end to end: a real +// NOTIFY_SOCKET, the daemon's own boot path announcing READY, and the run +// loop's own progress marker producing WATCHDOG heartbeats. +func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) { + listener := newNotifyListener(t) + t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms + + b, d := newNotifyTestBoot(t, 50*time.Millisecond) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var polling atomic.Bool + polling.Store(true) + go pollPasses(ctx, d, &polling, 5*time.Millisecond) + + b.announceReady() + b.startWatchdog(ctx) + + states := listener.states(t, 300*time.Millisecond) + if !slices.Contains(states, readyState) { + t.Errorf("states received = %v, want %q among them", states, readyState) + } + if countState(states, watchdogState) == 0 { + t.Errorf("states received = %v, want at least one %q among them", states, watchdogState) + } +} + +// TestWatchdogWithholdsHeartbeatWhenRunLoopStalls is the semantic that makes +// the heartbeat worth having: the beats stop once the loop stops making +// progress, so a wedged event loop trips systemd's watchdog instead of being +// kept alive by an independent ticker. +func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) { + listener := newNotifyListener(t) + t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms + + const cadence = 50 * time.Millisecond + b, d := newNotifyTestBoot(t, cadence) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var polling atomic.Bool + polling.Store(true) + go pollPasses(ctx, d, &polling, 5*time.Millisecond) + + b.startWatchdog(ctx) + if beats := countState(listener.states(t, 200*time.Millisecond), watchdogState); beats == 0 { + t.Fatal("no WATCHDOG=1 within 200ms while the run loop was polling") + } + + // From here the loop is inside a pass that never returns: nothing stamps + // the progress marker again. + polling.Store(false) + + // Discard the beats the last stamp still licenses. The marker counts as + // stale only once it is older than the loop's own cadence (50ms), and + // samples are 10ms apart, so 150ms covers every remaining beat with margin. + listener.states(t, 150*time.Millisecond) + + if late := listener.states(t, 300*time.Millisecond); len(late) != 0 { + t.Errorf("received %v after the run loop stalled, want no heartbeats", late) + } +} + +// TestNotifyUnsetIsNoOp pins the deployment contract: with no NOTIFY_SOCKET the +// process behaves exactly as it did before sd_notify existed. Docker, a plain +// terminal, and a systemd unit with no WatchdogSec all take this path, and none +// of them may see a datagram, a warning, or a failed start. +func TestNotifyUnsetIsNoOp(t *testing.T) { + t.Setenv(watchdogUsecEnv, "20000") + // t.Setenv cannot unset a variable: register the restoration first, then + // remove it for real so "unset" means unset. + t.Setenv(notifySocketEnv, "") + if err := os.Unsetenv(notifySocketEnv); err != nil { + t.Fatalf("unset %s: %v", notifySocketEnv, err) + } + + rec := ¬ifyRecorder{} + b, _ := newNotifyTestBoot(t, 50*time.Millisecond) + b.log = slog.New(rec) + + b.announceReady() + b.startWatchdog(t.Context()) + + if got := rec.all(); len(got) != 0 { + t.Errorf("with NOTIFY_SOCKET unset the notify path logged %d records, want none: %v", len(got), got) + } +} + +// TestNotifySocketFailureIsNotFatal pins the other half of the deployment +// contract: a NOTIFY_SOCKET that cannot be written to is logged and ignored. +// A daemon that refused to serve because systemd's socket was unavailable +// would be a regression on every deployment that has no systemd at all. +func TestNotifySocketFailureIsNotFatal(t *testing.T) { + missing := filepath.Join(t.TempDir(), "absent", "n.sock") + t.Setenv(notifySocketEnv, missing) + t.Setenv(watchdogUsecEnv, "20000") + + rec := ¬ifyRecorder{} + b, d := newNotifyTestBoot(t, 50*time.Millisecond) + b.log = slog.New(rec) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var polling atomic.Bool + polling.Store(true) + go pollPasses(ctx, d, &polling, 5*time.Millisecond) + + b.announceReady() + b.startWatchdog(ctx) + + if warns := rec.at(slog.LevelWarn); len(warns) != 1 { + t.Errorf("announceReady against an unusable socket logged %d warnings, want exactly 1", len(warns)) + } + + // The sampler keeps trying rather than dying on the first failure, so more + // warnings arrive while the loop progresses. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if len(rec.at(slog.LevelWarn)) > 1 { + break + } + time.Sleep(5 * time.Millisecond) + } + if warns := rec.at(slog.LevelWarn); len(warns) < 2 { + t.Errorf("the watchdog logged %d warnings, want the heartbeat still being attempted", len(warns)) + } + if errs := rec.at(slog.LevelError); len(errs) != 0 { + t.Errorf("an unusable notify socket logged %d errors, want none: %v", len(errs), errs) + } +}