diff --git a/internal/app/archied/gateway_test.go b/internal/app/archied/gateway_test.go index 024b815dd..7558c3c57 100644 --- a/internal/app/archied/gateway_test.go +++ b/internal/app/archied/gateway_test.go @@ -13,6 +13,7 @@ import ( pb "github.com/samcharles93/archie-core/internal/contracts/gateway/v1" "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/domain/taskactions" + "github.com/samcharles93/archie-core/internal/sdnotify" "github.com/samcharles93/archie-core/internal/taskstate" ) @@ -116,7 +117,7 @@ func TestGatewayServingAnnouncesReady(t *testing.T) { serveCh := make(chan error, 1) go func() { serveCh <- b.serveGatewayListener(ctx, ln, true, stubGatewayContract{}, nil) }() - if !listener.waitForState(t, 5*time.Second, readyState) { + if !listener.waitForState(t, 5*time.Second, sdnotify.ReadyState) { t.Fatal("no READY=1 datagram within 5s of the Gateway serving") } diff --git a/internal/app/archied/sd_notify.go b/internal/app/archied/sd_notify.go index ea42accec..6e2c2b409 100644 --- a/internal/app/archied/sd_notify.go +++ b/internal/app/archied/sd_notify.go @@ -1,22 +1,18 @@ -// sd_notify.go implements systemd's sd_notify(3) protocol for archied: one -// READY=1 when the run loop starts serving, and a WATCHDOG=1 heartbeat while -// the run loop is demonstrably still making progress. +// sd_notify.go wires the daemon into internal/sdnotify's implementation of +// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts +// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still +// making progress. // -// The protocol is three environment variables and one datagram per state, so -// this file speaks it directly instead of taking a module dependency: the whole -// client is net.DialUnix plus a Write. Nothing here is fatal -- a process with -// no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without WatchdogSec) -// sends nothing and serves exactly as before, and a socket that cannot be -// written to is logged and ignored. +// The protocol client and the heartbeat sampler live in internal/sdnotify +// because archie-messaging, a separate process that cannot import this +// package, is the next process that must be able to announce itself +// (archie-core-g5g8). What stays here is what is the daemon's own: which +// marker counts as progress (daemon.LastPollAt) and how long the loop may go +// without beginning a poll pass. // -// The heartbeat is deliberately not a bare ticker. A ticker fires whether or -// not the loop it stands for is alive, so it would keep systemd satisfied while -// the event loop was wedged -- the exact failure the watchdog exists to catch. -// The sampler instead reads the daemon's own progress marker -// (daemon.LastPollAt, stamped by the poll loop at the start of every pass) and -// withholds the heartbeat once that marker is older than the loop's own poll -// cadence. While passes keep beginning, beats flow at half of WatchdogSec; a -// loop blocked inside a pass stops being reported as alive. +// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal, +// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as +// before, and a socket that cannot be written to is logged and ignored. // // Consuming the states is the unit's business: READY=1 needs Type=notify, and // the heartbeat needs WatchdogSec. @@ -25,75 +21,12 @@ package archied import ( "context" "errors" - "fmt" - "log/slog" - "net" "os" - "strconv" "time" + + "github.com/samcharles93/archie-core/internal/sdnotify" ) -const ( - // notifySocketEnv names the socket a manager passes to a Type=notify unit. - notifySocketEnv = "NOTIFY_SOCKET" - // watchdogUsecEnv carries WatchdogSec in microseconds. - watchdogUsecEnv = "WATCHDOG_USEC" - // watchdogPIDEnv names the process whose beats the watchdog counts. - watchdogPIDEnv = "WATCHDOG_PID" - - // readyState and watchdogState are the two sd_notify states archied sends. - readyState = "READY=1" - watchdogState = "WATCHDOG=1" - - // watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half - // of WatchdogSec, so one missed beat is tolerated and two are not. - watchdogHeartbeatDivisor = 2 -) - -// notifier writes sd_notify states to the socket named in the environment. A -// notifier with no address -- the normal case away from systemd -- sends -// nothing and says nothing. -type notifier struct { - addr string - log *slog.Logger -} - -// newNotifier resolves the notify socket. The address is read from the -// environment systemd sets for the process, so systemd replaces the previous -// unit's socket with the new one across a restart. -func newNotifier(getenv func(string) string, log *slog.Logger) notifier { - return notifier{addr: getenv(notifySocketEnv), log: log} -} - -// send delivers one state line. A failure is logged, never returned: a -// notification is an observation about the daemon, and no observation is worth -// refusing to serve over. -func (n notifier) send(state string) { - if n.addr == "" { - return - } - if err := sendNotify(n.addr, state); err != nil { - n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err) - } -} - -// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is -// one datagram per state, so each send dials afresh: a manager restart recreates -// its socket, and a connection cached across one would fail every send after it. -// A leading '@' names the Linux abstract namespace, which user managers use; -// net.UnixAddr takes that syntax verbatim. -func sendNotify(addr, state string) error { - conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"}) - if err != nil { - return err - } - defer func() { _ = conn.Close() }() - if _, err := conn.Write([]byte(state)); err != nil { - return fmt.Errorf("write %q: %w", state, err) - } - return nil -} - // announceReady tells systemd the daemon has finished starting. It is called at // the moment a process declares itself serving: the daemon's run loop (next to // healthSurface.markServing), and the serve steps of archie-gateway and @@ -101,42 +34,35 @@ func sendNotify(addr, state string) error { // own and would otherwise hang a Type=notify unit until TimeoutStartSec while // perfectly healthy. func (b *boot) announceReady() { - notifyReady(b.log) -} - -// notifyReady is announceReady without the boot: the Gateway's serving tail -// runs on a boot it owns, but a free helper keeps the announcement one call -// rather than a second notion of "started" on each surface. -func notifyReady(log *slog.Logger) { - newNotifier(os.Getenv, log).send(readyState) + sdnotify.Ready(b.log) } // startWatchdog arms the heartbeat sampler for the daemon's run loop. It does // nothing unless a manager named a notify socket and asked for a watchdog // (WATCHDOG_USEC), so every deployment without systemd is unaffected. func (b *boot) startWatchdog(ctx context.Context) { - n := newNotifier(os.Getenv, b.log) - if n.addr == "" { + n := sdnotify.New(os.Getenv, b.log) + if !n.Enabled() { return } - interval, ok := watchdogHeartbeat(os.Getenv, os.Getpid(), b.log) + interval, ok := sdnotify.Heartbeat(os.Getenv, os.Getpid(), b.log) if !ok { return } - sampler := loopWatchdog{ - interval: interval, - cadence: b.loopCadence, - progress: b.d.LastPollAt, - beat: func() { n.send(watchdogState) }, - log: b.log, + sampler := sdnotify.Watchdog{ + Interval: interval, + Cadence: b.loopCadence, + Progress: b.d.LastPollAt, + Beat: func() { n.Send(sdnotify.WatchdogState) }, + Log: b.log, } b.log.Info("systemd watchdog armed", "heartbeat", interval.String(), "loop_cadence", b.loopCadence().String()) go func() { - if err := sampler.run(ctx); err != nil && !errors.Is(err, context.Canceled) { + if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { b.log.Error("systemd watchdog stopped", "err", err) } }() @@ -161,86 +87,3 @@ func (b *boot) loopCadence() time.Duration { } return cadence } - -// watchdogHeartbeat reads systemd's watchdog contract and returns the heartbeat -// cadence, which is half of WatchdogSec. ok is false when systemd asked for no -// watchdog, and also when it asked for one from a different process: -// WATCHDOG_PID names the process whose beats count, so a unit watching another -// process must not have this one beating on its behalf. -func watchdogHeartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) { - raw := getenv(watchdogUsecEnv) - if raw == "" { - return 0, false - } - if rawPID := getenv(watchdogPIDEnv); rawPID != "" { - wantPID, err := strconv.Atoi(rawPID) - if err != nil || wantPID != pid { - log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process", - "watchdog_pid", rawPID, "pid", pid, "err", err) - return 0, false - } - } - watchdog, err := time.ParseDuration(raw + "us") - if err != nil || watchdog <= 0 { - log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err) - return 0, false - } - return watchdog / watchdogHeartbeatDivisor, true -} - -// loopWatchdog samples the run loop's progress marker and emits the watchdog -// heartbeat only while that marker is fresh. Sampling is periodic; the beat is -// not, because it is licensed by the marker rather than by the clock. -type loopWatchdog struct { - // interval is the sampling cadence: half of systemd's WatchdogSec. - interval time.Duration - // cadence reports how long the loop may go between poll passes. - cadence func() time.Duration - // progress reports the start of the loop's most recent poll pass, or the - // zero time when no pass has begun yet. - progress func() time.Time - // beat emits one WATCHDOG=1. - beat func() - log *slog.Logger -} - -// run samples until ctx ends. It returns ctx.Err() on shutdown, which is the -// only way out: this sampler never gives up on its own, because a sampler that -// stopped would leave systemd watching a process nothing reports for. -func (w loopWatchdog) run(ctx context.Context) error { - armedAt := time.Now() - stalled := false - ticker := time.NewTicker(w.interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - } - - last := w.progress() - if last.IsZero() { - // No pass has begun at all. Measure staleness from arming, so a - // loop that never starts polling is caught like one that stopped. - last = armedAt - } - cadence := w.cadence() - switch age := time.Since(last); { - case age > cadence: - if !stalled { - w.log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does", - "stale_for", age.String(), "loop_cadence", cadence.String()) - stalled = true - } - default: - if stalled { - w.log.Info("systemd watchdog: run loop resumed; heartbeats resumed", - "last_pass_age", age.String()) - stalled = false - } - w.beat() - } - } -} diff --git a/internal/app/archied/sd_notify_test.go b/internal/app/archied/sd_notify_test.go index cf9e460f8..f371aa791 100644 --- a/internal/app/archied/sd_notify_test.go +++ b/internal/app/archied/sd_notify_test.go @@ -18,8 +18,16 @@ import ( "github.com/samcharles93/archie-core/internal/daemon" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest" + "github.com/samcharles93/archie-core/internal/sdnotify" ) +// The protocol-level properties of the notify client -- the '@' address form, +// the disabled and unusable socket contract, and the half-of-WatchdogSec +// heartbeat -- are pinned in internal/sdnotify, where the client now lives. +// These tests pin what is the daemon's: that its three serving entry points +// announce, that the sampler samples daemon.LastPollAt, and that the whole +// path stays silent and harmless without a NOTIFY_SOCKET. + // notifyRecorder captures every record at every level. The other recording // handler in this package keeps warnings only; these tests need "this path // logged nothing at all" as positive evidence that a disabled notifier stayed @@ -49,6 +57,7 @@ func (h *notifyRecorder) all() []slog.Record { return append([]slog.Record(nil), h.records...) } +// at returns the captured records at one level. func (h *notifyRecorder) at(level slog.Level) []slog.Record { h.mu.Lock() defer h.mu.Unlock() @@ -83,7 +92,7 @@ func listenNotify(t *testing.T, addr string) *notifyListener { t.Fatalf("listen for notify datagrams at %q: %v", addr, err) } t.Cleanup(func() { _ = conn.Close() }) - t.Setenv(notifySocketEnv, addr) + t.Setenv(sdnotify.NotifySocketEnv, addr) return ¬ifyListener{conn: conn} } @@ -174,93 +183,12 @@ func pollPasses(ctx context.Context, d *daemon.Daemon, on *atomic.Bool, every ti } } -// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not -// restricted to filesystem paths, and a client that stripped the prefix would -// connect to a file that does not exist instead of the abstract namespace. -func TestNotifyReachesAbstractSocket(t *testing.T) { - listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid())) - - (&boot{log: slog.New(slog.DiscardHandler)}).announceReady() - - if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, readyState) { - t.Errorf("states received = %v, want %q among them", states, readyState) - } -} - -// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the -// delivery contract: no socket means nothing is attempted and nothing is said, -// while a socket that cannot be written to is reported once and then ignored. -// Neither may ever be fatal. -func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) { - rec := ¬ifyRecorder{} - log := slog.New(rec) - - newNotifier(func(string) string { return "" }, log).send(readyState) - if got := rec.all(); len(got) != 0 { - t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got) - } - - missing := filepath.Join(t.TempDir(), "absent", "n.sock") - newNotifier(func(string) string { return missing }, log).send(readyState) - warns := rec.at(slog.LevelWarn) - if len(warns) != 1 { - t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns)) - } - if errs := rec.at(slog.LevelError); len(errs) != 0 { - t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs)) - } -} - -// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes -// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and -// WATCHDOG_PID names the process whose beats count. -func TestWatchdogHeartbeatInterval(t *testing.T) { - const self = 42 - tests := []struct { - name string - usec string - pidEnv string - want time.Duration - wantOK bool - }{ - {name: "no watchdog requested", wantOK: false}, - {name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true}, - {name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true}, - {name: "unparseable", usec: "soon", wantOK: false}, - {name: "zero", usec: "0", wantOK: false}, - {name: "negative", usec: "-1", wantOK: false}, - {name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true}, - {name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false}, - {name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - getenv := func(key string) string { - switch key { - case watchdogUsecEnv: - return tt.usec - case watchdogPIDEnv: - return tt.pidEnv - default: - return "" - } - } - got, ok := watchdogHeartbeat(getenv, self, slog.New(slog.DiscardHandler)) - if ok != tt.wantOK || got != tt.want { - t.Errorf("watchdogHeartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v", - tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK) - } - }) - } -} - // TestNotifyReadyAndWatchdogHeartbeat is the run-loop path end to end: a real // NOTIFY_SOCKET, the daemon's own boot path announcing READY, and the run // loop's own progress marker producing WATCHDOG heartbeats. func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) { listener := newNotifyListener(t) - t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms + t.Setenv(sdnotify.WatchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms b, d := newNotifyTestBoot(t, 50*time.Millisecond) ctx, cancel := context.WithCancel(t.Context()) @@ -274,11 +202,11 @@ func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) { b.startWatchdog(ctx) states := listener.states(t, 300*time.Millisecond) - if !slices.Contains(states, readyState) { - t.Errorf("states received = %v, want %q among them", states, readyState) + if !slices.Contains(states, sdnotify.ReadyState) { + t.Errorf("states received = %v, want %q among them", states, sdnotify.ReadyState) } - if countState(states, watchdogState) == 0 { - t.Errorf("states received = %v, want at least one %q among them", states, watchdogState) + if countState(states, sdnotify.WatchdogState) == 0 { + t.Errorf("states received = %v, want at least one %q among them", states, sdnotify.WatchdogState) } } @@ -288,7 +216,7 @@ func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) { // kept alive by an independent ticker. func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) { listener := newNotifyListener(t) - t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms + t.Setenv(sdnotify.WatchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms const cadence = 50 * time.Millisecond b, d := newNotifyTestBoot(t, cadence) @@ -300,7 +228,7 @@ func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) { go pollPasses(ctx, d, &polling, 5*time.Millisecond) b.startWatchdog(ctx) - if beats := countState(listener.states(t, 200*time.Millisecond), watchdogState); beats == 0 { + if beats := countState(listener.states(t, 200*time.Millisecond), sdnotify.WatchdogState); beats == 0 { t.Fatal("no WATCHDOG=1 within 200ms while the run loop was polling") } @@ -323,12 +251,12 @@ func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) { // terminal, and a systemd unit with no WatchdogSec all take this path, and none // of them may see a datagram, a warning, or a failed start. func TestNotifyUnsetIsNoOp(t *testing.T) { - t.Setenv(watchdogUsecEnv, "20000") + t.Setenv(sdnotify.WatchdogUsecEnv, "20000") // t.Setenv cannot unset a variable: register the restoration first, then // remove it for real so "unset" means unset. - t.Setenv(notifySocketEnv, "") - if err := os.Unsetenv(notifySocketEnv); err != nil { - t.Fatalf("unset %s: %v", notifySocketEnv, err) + t.Setenv(sdnotify.NotifySocketEnv, "") + if err := os.Unsetenv(sdnotify.NotifySocketEnv); err != nil { + t.Fatalf("unset %s: %v", sdnotify.NotifySocketEnv, err) } rec := ¬ifyRecorder{} @@ -349,8 +277,8 @@ func TestNotifyUnsetIsNoOp(t *testing.T) { // would be a regression on every deployment that has no systemd at all. func TestNotifySocketFailureIsNotFatal(t *testing.T) { missing := filepath.Join(t.TempDir(), "absent", "n.sock") - t.Setenv(notifySocketEnv, missing) - t.Setenv(watchdogUsecEnv, "20000") + t.Setenv(sdnotify.NotifySocketEnv, missing) + t.Setenv(sdnotify.WatchdogUsecEnv, "20000") rec := ¬ifyRecorder{} b, d := newNotifyTestBoot(t, 50*time.Millisecond) @@ -405,7 +333,7 @@ func TestStateStoreBootAnnouncesReady(t *testing.T) { errCh := make(chan error, 1) go func() { errCh <- RunStateStore(ctx, StateStoreOptions{Config: path, Listen: "127.0.0.1:0"}) }() - if !listener.waitForState(t, 15*time.Second, readyState) { + if !listener.waitForState(t, 15*time.Second, sdnotify.ReadyState) { select { case bootErr := <-errCh: t.Fatalf("RunStateStore exited before announcing ready: %v", bootErr) diff --git a/internal/app/archiemessaging/compose.go b/internal/app/archiemessaging/compose.go index 112752528..8380a6e18 100644 --- a/internal/app/archiemessaging/compose.go +++ b/internal/app/archiemessaging/compose.go @@ -17,6 +17,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/sdnotify" "github.com/samcharles93/archie-core/internal/secret" ) @@ -199,6 +200,18 @@ func (s *Service) Start(ctx context.Context) error { }) } + // Every configured channel has been handed its lifetime and the service + // blocks here serving them. systemd's READY=1 asserts the same fact this + // line has just established, so the announcement goes out here rather than + // from a second notion of "started" (the convention archie-core-1174 set + // for the daemon, the Gateway and the State Store; archie-core-g5g8 is this + // process joining them). Not earlier: a channel whose configuration is + // invalid fails composition, above, and READY must not precede it. Not + // later: a unit whose TimeoutStartSec expires first would restart a healthy + // process. Nothing is consumed until the unit says Type=notify, so this is + // inert on today's install. + sdnotify.Ready(s.log) + <-ctx.Done() for _, ch := range s.channels { diff --git a/internal/app/archiemessaging/readiness_notify_test.go b/internal/app/archiemessaging/readiness_notify_test.go new file mode 100644 index 000000000..814fce922 --- /dev/null +++ b/internal/app/archiemessaging/readiness_notify_test.go @@ -0,0 +1,99 @@ +package archiemessaging + +import ( + "context" + "errors" + "log/slog" + "net" + "path/filepath" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/sdnotify" +) + +// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket +// that collects the state lines the service sends. It is bound to the real +// NOTIFY_SOCKET for the duration of the test. +type notifyListener struct { + conn *net.UnixConn +} + +// listenNotify binds a notify socket in the test's temp dir and points +// NOTIFY_SOCKET at it. A unix socket path is capped near 108 bytes, so the +// short file name is what keeps the path inside t.TempDir() under that. +func listenNotify(t *testing.T) *notifyListener { + t.Helper() + addr := filepath.Join(t.TempDir(), "n.sock") + conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"}) + if err != nil { + t.Fatalf("listen for notify datagrams at %q: %v", addr, err) + } + t.Cleanup(func() { _ = conn.Close() }) + t.Setenv(sdnotify.NotifySocketEnv, addr) + return ¬ifyListener{conn: conn} +} + +// waitForState reads datagrams until one carries want or the window closes. +func (l *notifyListener) waitForState(t *testing.T, window time.Duration, want string) bool { + t.Helper() + deadline := time.Now().Add(window) + buf := make([]byte, 1024) + for time.Now().Before(deadline) { + if err := l.conn.SetReadDeadline(time.Now().Add(50 * time.Millisecond)); err != nil { + t.Fatalf("set notify read deadline: %v", err) + } + n, _, err := l.conn.ReadFromUnix(buf) + if err != nil { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + continue + } + t.Fatalf("read notify datagram: %v", err) + } + if string(buf[:n]) == want { + return true + } + } + return false +} + +// TestServiceAnnouncesReadyWhenServing: archie-messaging run as a Type=notify +// unit must send READY=1 at the moment it declares itself serving, or systemd +// kills a healthy process once TimeoutStartSec expires. It could not before: +// the client lived in internal/app/archied, which this process cannot import +// (archie-core-g5g8). The announcement is judged against the same evidence the +// other serve paths use -- the process itself says it is serving -- and not +// against a second notion of started invented here. +func TestServiceAnnouncesReadyWhenServing(t *testing.T) { + listener := listenNotify(t) + + srv, err := compose(t.Context(), deps{ + Config: ResolvedConfig{Options: Options{ShutdownTimeout: 2 * time.Second}}, + Log: slog.New(slog.DiscardHandler), + Chat: &dummyChatContract{}, + }) + if err != nil { + t.Fatalf("compose: %v", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + started := make(chan error, 1) + go func() { started <- srv.Start(ctx) }() + t.Cleanup(func() { + cancel() + srv.Stop() + select { + case err := <-started: + if err != nil { + t.Errorf("srv.Start returned %v, want a clean stop", err) + } + case <-time.After(3 * time.Second): + t.Error("srv.Start did not return after Stop, so a channel leaked its goroutine") + } + }) + + if !listener.waitForState(t, 5*time.Second, sdnotify.ReadyState) { + t.Fatal("no READY=1 datagram within 5s of the Messaging Service serving") + } +} diff --git a/internal/sdnotify/sdnotify.go b/internal/sdnotify/sdnotify.go new file mode 100644 index 000000000..536d42eac --- /dev/null +++ b/internal/sdnotify/sdnotify.go @@ -0,0 +1,205 @@ +// Package sdnotify implements systemd's sd_notify(3) protocol: one READY=1 when +// a process starts serving, and a WATCHDOG=1 heartbeat while the loop it stands +// for is demonstrably still making progress. +// +// The protocol is three environment variables and one datagram per state, so +// this package speaks it directly instead of taking a module dependency: the +// whole client is net.DialUnix plus a Write. Nothing here is fatal -- a process +// with no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without +// WatchdogSec) sends nothing and serves exactly as before, and a socket that +// cannot be written to is logged and ignored. +// +// It is cross-cutting per docs/architecture/organisation.md: any layer and any +// binary may import it, and it imports nothing of ours. It lives outside a +// composition package because a process that must announce itself cannot +// import another process's composition -- archied and archie-messaging are two +// processes that announce, and only one of them can own archied's boot +// (archie-core-g5g8). +// +// What stays with the caller is everything that is the caller's: which marker +// counts as progress, how long its loop may go without a pass, and the point +// at which it considers itself serving. +// +// The heartbeat is deliberately not a bare ticker. A ticker fires whether or +// not the loop it stands for is alive, so it would keep systemd satisfied while +// the event loop was wedged -- the exact failure the watchdog exists to catch. +// The sampler instead reads the caller's own progress marker through Progress +// and withholds the heartbeat once that marker is older than the cadence +// Cadence reports. While passes keep beginning, beats flow at half of +// WatchdogSec; a loop blocked inside a pass stops being reported as alive. +// +// Consuming the states is the unit's business: READY=1 needs Type=notify, and +// the heartbeat needs WatchdogSec. +package sdnotify + +import ( + "context" + "fmt" + "log/slog" + "net" + "os" + "strconv" + "time" +) + +const ( + // NotifySocketEnv names the socket a manager passes to a Type=notify unit. + NotifySocketEnv = "NOTIFY_SOCKET" + // WatchdogUsecEnv carries WatchdogSec in microseconds. + WatchdogUsecEnv = "WATCHDOG_USEC" + // WatchdogPIDEnv names the process whose beats the watchdog counts. + WatchdogPIDEnv = "WATCHDOG_PID" + + // ReadyState and WatchdogState are the two sd_notify states a process + // announces: it has started serving, and it is still making progress. + ReadyState = "READY=1" + WatchdogState = "WATCHDOG=1" + + // watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half + // of WatchdogSec, so one missed beat is tolerated and two are not. + watchdogHeartbeatDivisor = 2 +) + +// Notifier writes sd_notify states to the socket named in the environment. A +// notifier with no address -- the normal case away from systemd -- sends +// nothing and says nothing. +type Notifier struct { + addr string + log *slog.Logger +} + +// New resolves the notify socket. The address is read from the environment +// systemd sets for the process, so systemd replaces the previous unit's socket +// with the new one across a restart. +func New(getenv func(string) string, log *slog.Logger) Notifier { + return Notifier{addr: getenv(NotifySocketEnv), log: log} +} + +// Enabled reports whether a manager named a notify socket at all. It is the +// one fact a caller needs to decide whether a heartbeat is worth arming: with +// no socket every send is a no-op, and a sampler that cannot deliver a beat +// should not exist to withhold one. +func (n Notifier) Enabled() bool { return n.addr != "" } + +// Send delivers one state line. A failure is logged, never returned: a +// notification is an observation about the process, and no observation is +// worth refusing to serve over. +func (n Notifier) Send(state string) { + if n.addr == "" { + return + } + if err := sendNotify(n.addr, state); err != nil { + n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err) + } +} + +// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is +// one datagram per state, so each send dials afresh: a manager restart recreates +// its socket, and a connection cached across one would fail every send after it. +// A leading '@' names the Linux abstract namespace, which user managers use; +// net.UnixAddr takes that syntax verbatim. +func sendNotify(addr, state string) error { + conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"}) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + if _, err := conn.Write([]byte(state)); err != nil { + return fmt.Errorf("write %q: %w", state, err) + } + return nil +} + +// Ready sends READY=1 to the socket named in the process environment. It is +// the one call a composing package needs to declare itself serving, and it +// belongs at the moment the process already says so: the daemon's run loop +// (next to its own health surface), and the serve steps of archie-gateway, +// archie-state-store and archie-messaging, which have readiness of their own +// and would otherwise hang a Type=notify unit until TimeoutStartSec while +// perfectly healthy (archie-core-1174, archie-core-g5g8). +func Ready(log *slog.Logger) { + New(os.Getenv, log).Send(ReadyState) +} + +// Heartbeat reads systemd's watchdog contract and returns the heartbeat +// cadence, which is half of WatchdogSec. ok is false when systemd asked for no +// watchdog, and also when it asked for one from a different process: +// WATCHDOG_PID names the process whose beats count, so a unit watching another +// process must not have this one beating on its behalf. +func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) { + raw := getenv(WatchdogUsecEnv) + if raw == "" { + return 0, false + } + if rawPID := getenv(WatchdogPIDEnv); rawPID != "" { + wantPID, err := strconv.Atoi(rawPID) + if err != nil || wantPID != pid { + log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process", + "watchdog_pid", rawPID, "pid", pid, "err", err) + return 0, false + } + } + watchdog, err := time.ParseDuration(raw + "us") + if err != nil || watchdog <= 0 { + log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err) + return 0, false + } + return watchdog / watchdogHeartbeatDivisor, true +} + +// Watchdog samples the caller's progress marker and emits the watchdog +// heartbeat only while that marker is fresh. Sampling is periodic; the beat is +// not, because it is licensed by the marker rather than by the clock. +type Watchdog struct { + // Interval is the sampling cadence: half of systemd's WatchdogSec. + Interval time.Duration + // Cadence reports how long the loop may go between passes. + Cadence func() time.Duration + // Progress reports the start of the loop's most recent pass, or the zero + // time when no pass has begun yet. + Progress func() time.Time + // Beat emits one WATCHDOG=1. + Beat func() + Log *slog.Logger +} + +// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the +// only way out: this sampler never gives up on its own, because a sampler that +// stopped would leave systemd watching a process nothing reports for. +func (w Watchdog) Run(ctx context.Context) error { + armedAt := time.Now() + stalled := false + ticker := time.NewTicker(w.Interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + + last := w.Progress() + if last.IsZero() { + // No pass has begun at all. Measure staleness from arming, so a + // loop that never starts polling is caught like one that stopped. + last = armedAt + } + cadence := w.Cadence() + switch age := time.Since(last); { + case age > cadence: + if !stalled { + w.Log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does", + "stale_for", age.String(), "loop_cadence", cadence.String()) + stalled = true + } + default: + if stalled { + w.Log.Info("systemd watchdog: run loop resumed; heartbeats resumed", + "last_pass_age", age.String()) + stalled = false + } + w.Beat() + } + } +} diff --git a/internal/sdnotify/sdnotify_test.go b/internal/sdnotify/sdnotify_test.go new file mode 100644 index 000000000..0f4226cdd --- /dev/null +++ b/internal/sdnotify/sdnotify_test.go @@ -0,0 +1,178 @@ +package sdnotify + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "os" + "path/filepath" + "slices" + "sync" + "testing" + "time" +) + +// notifyRecorder captures every record at every level. A warnings-only +// recorder would not do: these tests need "this path logged nothing at all" as +// positive evidence that a disabled notifier stayed silent, so the level has +// to be part of what is captured. +type notifyRecorder struct { + mu sync.Mutex + records []slog.Record +} + +func (h *notifyRecorder) Enabled(context.Context, slog.Level) bool { return true } + +func (h *notifyRecorder) Handle(_ context.Context, r slog.Record) error { + // The Record is the caller's to reuse once Handle returns, so keep a copy. + rec := r.Clone() + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, rec) + return nil +} + +func (h *notifyRecorder) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *notifyRecorder) WithGroup(string) slog.Handler { return h } + +func (h *notifyRecorder) all() []slog.Record { + h.mu.Lock() + defer h.mu.Unlock() + return append([]slog.Record(nil), h.records...) +} + +func (h *notifyRecorder) at(level slog.Level) []slog.Record { + h.mu.Lock() + defer h.mu.Unlock() + var out []slog.Record + for _, r := range h.records { + if r.Level == level { + out = append(out, r) + } + } + return out +} + +// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket +// that collects the state lines a process sends. It is bound to the real +// NOTIFY_SOCKET for the duration of the test. +type notifyListener struct { + conn *net.UnixConn +} + +// listenNotify binds a notify socket at addr and points NOTIFY_SOCKET at it. +func listenNotify(t *testing.T, addr string) *notifyListener { + t.Helper() + conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"}) + if err != nil { + t.Fatalf("listen for notify datagrams at %q: %v", addr, err) + } + t.Cleanup(func() { _ = conn.Close() }) + t.Setenv(NotifySocketEnv, addr) + return ¬ifyListener{conn: conn} +} + +// states drains every state line received within window. +func (l *notifyListener) states(t *testing.T, window time.Duration) []string { + t.Helper() + deadline := time.Now().Add(window) + var got []string + buf := make([]byte, 1024) + for { + if err := l.conn.SetReadDeadline(deadline); err != nil { + t.Fatalf("set notify read deadline: %v", err) + } + n, _, err := l.conn.ReadFromUnix(buf) + if err != nil { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return got + } + t.Fatalf("read notify datagram: %v", err) + } + got = append(got, string(buf[:n])) + } +} + +// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not +// restricted to filesystem paths, and a client that stripped the prefix would +// connect to a file that does not exist instead of the abstract namespace. +func TestNotifyReachesAbstractSocket(t *testing.T) { + listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid())) + + Ready(slog.New(slog.DiscardHandler)) + + if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, ReadyState) { + t.Errorf("states received = %v, want %q among them", states, ReadyState) + } +} + +// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the +// delivery contract: no socket means nothing is attempted and nothing is said, +// while a socket that cannot be written to is reported once and then ignored. +// Neither may ever be fatal. +func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) { + rec := ¬ifyRecorder{} + log := slog.New(rec) + + New(func(string) string { return "" }, log).Send(ReadyState) + if got := rec.all(); len(got) != 0 { + t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got) + } + + missing := filepath.Join(t.TempDir(), "absent", "n.sock") + New(func(string) string { return missing }, log).Send(ReadyState) + warns := rec.at(slog.LevelWarn) + if len(warns) != 1 { + t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns)) + } + if errs := rec.at(slog.LevelError); len(errs) != 0 { + t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs)) + } +} + +// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes +// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and +// WATCHDOG_PID names the process whose beats count. +func TestWatchdogHeartbeatInterval(t *testing.T) { + const self = 42 + tests := []struct { + name string + usec string + pidEnv string + want time.Duration + wantOK bool + }{ + {name: "no watchdog requested", wantOK: false}, + {name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true}, + {name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true}, + {name: "unparseable", usec: "soon", wantOK: false}, + {name: "zero", usec: "0", wantOK: false}, + {name: "negative", usec: "-1", wantOK: false}, + {name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true}, + {name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false}, + {name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + getenv := func(key string) string { + switch key { + case WatchdogUsecEnv: + return tt.usec + case WatchdogPIDEnv: + return tt.pidEnv + default: + return "" + } + } + got, ok := Heartbeat(getenv, self, slog.New(slog.DiscardHandler)) + if ok != tt.wantOK || got != tt.want { + t.Errorf("Heartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v", + tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK) + } + }) + } +}