refactor(archied): move the sd_notify client to internal/sdnotify

The sd_notify(3) client lived in internal/app/archied, so archie-messaging --
a process that must not import archied's composition -- could never announce
itself to systemd, and could therefore never be a Type=notify unit
(archie-core-g5g8).

The protocol client and the heartbeat sampler move to internal/sdnotify, a
cross-cutting leaf that imports nothing of ours. What stays in archied is
what is the daemon's own: which marker counts as progress (daemon.LastPollAt)
and how long the run loop may go between poll passes. The three entry points
that already announced keep announcing, unchanged.

The Messaging Service announces READY=1 at the point it declares itself
serving, the way the daemon, the Gateway and the State Store already do. No
unit is switched to Type=notify here: that is a separate decision, and
archie-messaging.service is still Type=simple.
This commit is contained in:
sam committed 2026-09-30 11:23:33 +10:00
1 parent b314470281
commit 275d337032
7 files changed
+547 -280

No files matched your search

+2 -1
View File
@@ -13,6 +13,7 @@ import (
pb "github.com/samcharles93/archie-core/internal/contracts/gateway/v1"
"github.com/samcharles93/archie-core/internal/domain/messaging"
"github.com/samcharles93/archie-core/internal/domain/taskactions"
"github.com/samcharles93/archie-core/internal/sdnotify"
"github.com/samcharles93/archie-core/internal/taskstate"
)
@@ -116,7 +117,7 @@ func TestGatewayServingAnnouncesReady(t *testing.T) {
serveCh := make(chan error, 1)
go func() { serveCh <- b.serveGatewayListener(ctx, ln, true, stubGatewayContract{}, nil) }()
if !listener.waitForState(t, 5*time.Second, readyState) {
if !listener.waitForState(t, 5*time.Second, sdnotify.ReadyState) {
t.Fatal("no READY=1 datagram within 5s of the Gateway serving")
}
+26 -183
View File
@@ -1,22 +1,18 @@
// sd_notify.go implements systemd's sd_notify(3) protocol for archied: one
// READY=1 when the run loop starts serving, and a WATCHDOG=1 heartbeat while
// the run loop is demonstrably still making progress.
// sd_notify.go wires the daemon into internal/sdnotify's implementation of
// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts
// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still
// making progress.
//
// The protocol is three environment variables and one datagram per state, so
// this file speaks it directly instead of taking a module dependency: the whole
// client is net.DialUnix plus a Write. Nothing here is fatal -- a process with
// no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without WatchdogSec)
// sends nothing and serves exactly as before, and a socket that cannot be
// written to is logged and ignored.
// The protocol client and the heartbeat sampler live in internal/sdnotify
// because archie-messaging, a separate process that cannot import this
// package, is the next process that must be able to announce itself
// (archie-core-g5g8). What stays here is what is the daemon's own: which
// marker counts as progress (daemon.LastPollAt) and how long the loop may go
// without beginning a poll pass.
//
// The heartbeat is deliberately not a bare ticker. A ticker fires whether or
// not the loop it stands for is alive, so it would keep systemd satisfied while
// the event loop was wedged -- the exact failure the watchdog exists to catch.
// The sampler instead reads the daemon's own progress marker
// (daemon.LastPollAt, stamped by the poll loop at the start of every pass) and
// withholds the heartbeat once that marker is older than the loop's own poll
// cadence. While passes keep beginning, beats flow at half of WatchdogSec; a
// loop blocked inside a pass stops being reported as alive.
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as
// before, and a socket that cannot be written to is logged and ignored.
//
// Consuming the states is the unit's business: READY=1 needs Type=notify, and
// the heartbeat needs WatchdogSec.
@@ -25,75 +21,12 @@ package archied
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"os"
"strconv"
"time"
"github.com/samcharles93/archie-core/internal/sdnotify"
)
const (
// notifySocketEnv names the socket a manager passes to a Type=notify unit.
notifySocketEnv = "NOTIFY_SOCKET"
// watchdogUsecEnv carries WatchdogSec in microseconds.
watchdogUsecEnv = "WATCHDOG_USEC"
// watchdogPIDEnv names the process whose beats the watchdog counts.
watchdogPIDEnv = "WATCHDOG_PID"
// readyState and watchdogState are the two sd_notify states archied sends.
readyState = "READY=1"
watchdogState = "WATCHDOG=1"
// watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half
// of WatchdogSec, so one missed beat is tolerated and two are not.
watchdogHeartbeatDivisor = 2
)
// notifier writes sd_notify states to the socket named in the environment. A
// notifier with no address -- the normal case away from systemd -- sends
// nothing and says nothing.
type notifier struct {
addr string
log *slog.Logger
}
// newNotifier resolves the notify socket. The address is read from the
// environment systemd sets for the process, so systemd replaces the previous
// unit's socket with the new one across a restart.
func newNotifier(getenv func(string) string, log *slog.Logger) notifier {
return notifier{addr: getenv(notifySocketEnv), log: log}
}
// send delivers one state line. A failure is logged, never returned: a
// notification is an observation about the daemon, and no observation is worth
// refusing to serve over.
func (n notifier) send(state string) {
if n.addr == "" {
return
}
if err := sendNotify(n.addr, state); err != nil {
n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err)
}
}
// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is
// one datagram per state, so each send dials afresh: a manager restart recreates
// its socket, and a connection cached across one would fail every send after it.
// A leading '@' names the Linux abstract namespace, which user managers use;
// net.UnixAddr takes that syntax verbatim.
func sendNotify(addr, state string) error {
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"})
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
if _, err := conn.Write([]byte(state)); err != nil {
return fmt.Errorf("write %q: %w", state, err)
}
return nil
}
// announceReady tells systemd the daemon has finished starting. It is called at
// the moment a process declares itself serving: the daemon's run loop (next to
// healthSurface.markServing), and the serve steps of archie-gateway and
@@ -101,42 +34,35 @@ func sendNotify(addr, state string) error {
// own and would otherwise hang a Type=notify unit until TimeoutStartSec while
// perfectly healthy.
func (b *boot) announceReady() {
notifyReady(b.log)
}
// notifyReady is announceReady without the boot: the Gateway's serving tail
// runs on a boot it owns, but a free helper keeps the announcement one call
// rather than a second notion of "started" on each surface.
func notifyReady(log *slog.Logger) {
newNotifier(os.Getenv, log).send(readyState)
sdnotify.Ready(b.log)
}
// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does
// nothing unless a manager named a notify socket and asked for a watchdog
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
func (b *boot) startWatchdog(ctx context.Context) {
n := newNotifier(os.Getenv, b.log)
if n.addr == "" {
n := sdnotify.New(os.Getenv, b.log)
if !n.Enabled() {
return
}
interval, ok := watchdogHeartbeat(os.Getenv, os.Getpid(), b.log)
interval, ok := sdnotify.Heartbeat(os.Getenv, os.Getpid(), b.log)
if !ok {
return
}
sampler := loopWatchdog{
interval: interval,
cadence: b.loopCadence,
progress: b.d.LastPollAt,
beat: func() { n.send(watchdogState) },
log: b.log,
sampler := sdnotify.Watchdog{
Interval: interval,
Cadence: b.loopCadence,
Progress: b.d.LastPollAt,
Beat: func() { n.Send(sdnotify.WatchdogState) },
Log: b.log,
}
b.log.Info("systemd watchdog armed",
"heartbeat", interval.String(),
"loop_cadence", b.loopCadence().String())
go func() {
if err := sampler.run(ctx); err != nil && !errors.Is(err, context.Canceled) {
if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
b.log.Error("systemd watchdog stopped", "err", err)
}
}()
@@ -161,86 +87,3 @@ func (b *boot) loopCadence() time.Duration {
}
return cadence
}
// watchdogHeartbeat reads systemd's watchdog contract and returns the heartbeat
// cadence, which is half of WatchdogSec. ok is false when systemd asked for no
// watchdog, and also when it asked for one from a different process:
// WATCHDOG_PID names the process whose beats count, so a unit watching another
// process must not have this one beating on its behalf.
func watchdogHeartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) {
raw := getenv(watchdogUsecEnv)
if raw == "" {
return 0, false
}
if rawPID := getenv(watchdogPIDEnv); rawPID != "" {
wantPID, err := strconv.Atoi(rawPID)
if err != nil || wantPID != pid {
log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process",
"watchdog_pid", rawPID, "pid", pid, "err", err)
return 0, false
}
}
watchdog, err := time.ParseDuration(raw + "us")
if err != nil || watchdog <= 0 {
log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err)
return 0, false
}
return watchdog / watchdogHeartbeatDivisor, true
}
// loopWatchdog samples the run loop's progress marker and emits the watchdog
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
// not, because it is licensed by the marker rather than by the clock.
type loopWatchdog struct {
// interval is the sampling cadence: half of systemd's WatchdogSec.
interval time.Duration
// cadence reports how long the loop may go between poll passes.
cadence func() time.Duration
// progress reports the start of the loop's most recent poll pass, or the
// zero time when no pass has begun yet.
progress func() time.Time
// beat emits one WATCHDOG=1.
beat func()
log *slog.Logger
}
// run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
// only way out: this sampler never gives up on its own, because a sampler that
// stopped would leave systemd watching a process nothing reports for.
func (w loopWatchdog) run(ctx context.Context) error {
armedAt := time.Now()
stalled := false
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
last := w.progress()
if last.IsZero() {
// No pass has begun at all. Measure staleness from arming, so a
// loop that never starts polling is caught like one that stopped.
last = armedAt
}
cadence := w.cadence()
switch age := time.Since(last); {
case age > cadence:
if !stalled {
w.log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
"stale_for", age.String(), "loop_cadence", cadence.String())
stalled = true
}
default:
if stalled {
w.log.Info("systemd watchdog: run loop resumed; heartbeats resumed",
"last_pass_age", age.String())
stalled = false
}
w.beat()
}
}
}
+24 -96
View File
@@ -18,8 +18,16 @@ import (
"github.com/samcharles93/archie-core/internal/daemon"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgtest"
"github.com/samcharles93/archie-core/internal/sdnotify"
)
// The protocol-level properties of the notify client -- the '@' address form,
// the disabled and unusable socket contract, and the half-of-WatchdogSec
// heartbeat -- are pinned in internal/sdnotify, where the client now lives.
// These tests pin what is the daemon's: that its three serving entry points
// announce, that the sampler samples daemon.LastPollAt, and that the whole
// path stays silent and harmless without a NOTIFY_SOCKET.
// notifyRecorder captures every record at every level. The other recording
// handler in this package keeps warnings only; these tests need "this path
// logged nothing at all" as positive evidence that a disabled notifier stayed
@@ -49,6 +57,7 @@ func (h *notifyRecorder) all() []slog.Record {
return append([]slog.Record(nil), h.records...)
}
// at returns the captured records at one level.
func (h *notifyRecorder) at(level slog.Level) []slog.Record {
h.mu.Lock()
defer h.mu.Unlock()
@@ -83,7 +92,7 @@ func listenNotify(t *testing.T, addr string) *notifyListener {
t.Fatalf("listen for notify datagrams at %q: %v", addr, err)
}
t.Cleanup(func() { _ = conn.Close() })
t.Setenv(notifySocketEnv, addr)
t.Setenv(sdnotify.NotifySocketEnv, addr)
return &notifyListener{conn: conn}
}
@@ -174,93 +183,12 @@ func pollPasses(ctx context.Context, d *daemon.Daemon, on *atomic.Bool, every ti
}
}
// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not
// restricted to filesystem paths, and a client that stripped the prefix would
// connect to a file that does not exist instead of the abstract namespace.
func TestNotifyReachesAbstractSocket(t *testing.T) {
listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid()))
(&boot{log: slog.New(slog.DiscardHandler)}).announceReady()
if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, readyState) {
t.Errorf("states received = %v, want %q among them", states, readyState)
}
}
// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the
// delivery contract: no socket means nothing is attempted and nothing is said,
// while a socket that cannot be written to is reported once and then ignored.
// Neither may ever be fatal.
func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) {
rec := &notifyRecorder{}
log := slog.New(rec)
newNotifier(func(string) string { return "" }, log).send(readyState)
if got := rec.all(); len(got) != 0 {
t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got)
}
missing := filepath.Join(t.TempDir(), "absent", "n.sock")
newNotifier(func(string) string { return missing }, log).send(readyState)
warns := rec.at(slog.LevelWarn)
if len(warns) != 1 {
t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns))
}
if errs := rec.at(slog.LevelError); len(errs) != 0 {
t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs))
}
}
// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes
// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and
// WATCHDOG_PID names the process whose beats count.
func TestWatchdogHeartbeatInterval(t *testing.T) {
const self = 42
tests := []struct {
name string
usec string
pidEnv string
want time.Duration
wantOK bool
}{
{name: "no watchdog requested", wantOK: false},
{name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true},
{name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true},
{name: "unparseable", usec: "soon", wantOK: false},
{name: "zero", usec: "0", wantOK: false},
{name: "negative", usec: "-1", wantOK: false},
{name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true},
{name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false},
{name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
getenv := func(key string) string {
switch key {
case watchdogUsecEnv:
return tt.usec
case watchdogPIDEnv:
return tt.pidEnv
default:
return ""
}
}
got, ok := watchdogHeartbeat(getenv, self, slog.New(slog.DiscardHandler))
if ok != tt.wantOK || got != tt.want {
t.Errorf("watchdogHeartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v",
tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK)
}
})
}
}
// TestNotifyReadyAndWatchdogHeartbeat is the run-loop path end to end: a real
// NOTIFY_SOCKET, the daemon's own boot path announcing READY, and the run
// loop's own progress marker producing WATCHDOG heartbeats.
func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) {
listener := newNotifyListener(t)
t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
t.Setenv(sdnotify.WatchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
b, d := newNotifyTestBoot(t, 50*time.Millisecond)
ctx, cancel := context.WithCancel(t.Context())
@@ -274,11 +202,11 @@ func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) {
b.startWatchdog(ctx)
states := listener.states(t, 300*time.Millisecond)
if !slices.Contains(states, readyState) {
t.Errorf("states received = %v, want %q among them", states, readyState)
if !slices.Contains(states, sdnotify.ReadyState) {
t.Errorf("states received = %v, want %q among them", states, sdnotify.ReadyState)
}
if countState(states, watchdogState) == 0 {
t.Errorf("states received = %v, want at least one %q among them", states, watchdogState)
if countState(states, sdnotify.WatchdogState) == 0 {
t.Errorf("states received = %v, want at least one %q among them", states, sdnotify.WatchdogState)
}
}
@@ -288,7 +216,7 @@ func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) {
// kept alive by an independent ticker.
func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) {
listener := newNotifyListener(t)
t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
t.Setenv(sdnotify.WatchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
const cadence = 50 * time.Millisecond
b, d := newNotifyTestBoot(t, cadence)
@@ -300,7 +228,7 @@ func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) {
go pollPasses(ctx, d, &polling, 5*time.Millisecond)
b.startWatchdog(ctx)
if beats := countState(listener.states(t, 200*time.Millisecond), watchdogState); beats == 0 {
if beats := countState(listener.states(t, 200*time.Millisecond), sdnotify.WatchdogState); beats == 0 {
t.Fatal("no WATCHDOG=1 within 200ms while the run loop was polling")
}
@@ -323,12 +251,12 @@ func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) {
// terminal, and a systemd unit with no WatchdogSec all take this path, and none
// of them may see a datagram, a warning, or a failed start.
func TestNotifyUnsetIsNoOp(t *testing.T) {
t.Setenv(watchdogUsecEnv, "20000")
t.Setenv(sdnotify.WatchdogUsecEnv, "20000")
// t.Setenv cannot unset a variable: register the restoration first, then
// remove it for real so "unset" means unset.
t.Setenv(notifySocketEnv, "")
if err := os.Unsetenv(notifySocketEnv); err != nil {
t.Fatalf("unset %s: %v", notifySocketEnv, err)
t.Setenv(sdnotify.NotifySocketEnv, "")
if err := os.Unsetenv(sdnotify.NotifySocketEnv); err != nil {
t.Fatalf("unset %s: %v", sdnotify.NotifySocketEnv, err)
}
rec := &notifyRecorder{}
@@ -349,8 +277,8 @@ func TestNotifyUnsetIsNoOp(t *testing.T) {
// would be a regression on every deployment that has no systemd at all.
func TestNotifySocketFailureIsNotFatal(t *testing.T) {
missing := filepath.Join(t.TempDir(), "absent", "n.sock")
t.Setenv(notifySocketEnv, missing)
t.Setenv(watchdogUsecEnv, "20000")
t.Setenv(sdnotify.NotifySocketEnv, missing)
t.Setenv(sdnotify.WatchdogUsecEnv, "20000")
rec := &notifyRecorder{}
b, d := newNotifyTestBoot(t, 50*time.Millisecond)
@@ -405,7 +333,7 @@ func TestStateStoreBootAnnouncesReady(t *testing.T) {
errCh := make(chan error, 1)
go func() { errCh <- RunStateStore(ctx, StateStoreOptions{Config: path, Listen: "127.0.0.1:0"}) }()
if !listener.waitForState(t, 15*time.Second, readyState) {
if !listener.waitForState(t, 15*time.Second, sdnotify.ReadyState) {
select {
case bootErr := <-errCh:
t.Fatalf("RunStateStore exited before announcing ready: %v", bootErr)
+13
View File
@@ -17,6 +17,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/domain/messaging"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/sdnotify"
"github.com/samcharles93/archie-core/internal/secret"
)
@@ -199,6 +200,18 @@ func (s *Service) Start(ctx context.Context) error {
})
}
// Every configured channel has been handed its lifetime and the service
// blocks here serving them. systemd's READY=1 asserts the same fact this
// line has just established, so the announcement goes out here rather than
// from a second notion of "started" (the convention archie-core-1174 set
// for the daemon, the Gateway and the State Store; archie-core-g5g8 is this
// process joining them). Not earlier: a channel whose configuration is
// invalid fails composition, above, and READY must not precede it. Not
// later: a unit whose TimeoutStartSec expires first would restart a healthy
// process. Nothing is consumed until the unit says Type=notify, so this is
// inert on today's install.
sdnotify.Ready(s.log)
<-ctx.Done()
for _, ch := range s.channels {
@@ -0,0 +1,99 @@
package archiemessaging
import (
"context"
"errors"
"log/slog"
"net"
"path/filepath"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/sdnotify"
)
// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket
// that collects the state lines the service sends. It is bound to the real
// NOTIFY_SOCKET for the duration of the test.
type notifyListener struct {
conn *net.UnixConn
}
// listenNotify binds a notify socket in the test's temp dir and points
// NOTIFY_SOCKET at it. A unix socket path is capped near 108 bytes, so the
// short file name is what keeps the path inside t.TempDir() under that.
func listenNotify(t *testing.T) *notifyListener {
t.Helper()
addr := filepath.Join(t.TempDir(), "n.sock")
conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"})
if err != nil {
t.Fatalf("listen for notify datagrams at %q: %v", addr, err)
}
t.Cleanup(func() { _ = conn.Close() })
t.Setenv(sdnotify.NotifySocketEnv, addr)
return &notifyListener{conn: conn}
}
// waitForState reads datagrams until one carries want or the window closes.
func (l *notifyListener) waitForState(t *testing.T, window time.Duration, want string) bool {
t.Helper()
deadline := time.Now().Add(window)
buf := make([]byte, 1024)
for time.Now().Before(deadline) {
if err := l.conn.SetReadDeadline(time.Now().Add(50 * time.Millisecond)); err != nil {
t.Fatalf("set notify read deadline: %v", err)
}
n, _, err := l.conn.ReadFromUnix(buf)
if err != nil {
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
continue
}
t.Fatalf("read notify datagram: %v", err)
}
if string(buf[:n]) == want {
return true
}
}
return false
}
// TestServiceAnnouncesReadyWhenServing: archie-messaging run as a Type=notify
// unit must send READY=1 at the moment it declares itself serving, or systemd
// kills a healthy process once TimeoutStartSec expires. It could not before:
// the client lived in internal/app/archied, which this process cannot import
// (archie-core-g5g8). The announcement is judged against the same evidence the
// other serve paths use -- the process itself says it is serving -- and not
// against a second notion of started invented here.
func TestServiceAnnouncesReadyWhenServing(t *testing.T) {
listener := listenNotify(t)
srv, err := compose(t.Context(), deps{
Config: ResolvedConfig{Options: Options{ShutdownTimeout: 2 * time.Second}},
Log: slog.New(slog.DiscardHandler),
Chat: &dummyChatContract{},
})
if err != nil {
t.Fatalf("compose: %v", err)
}
ctx, cancel := context.WithCancel(t.Context())
started := make(chan error, 1)
go func() { started <- srv.Start(ctx) }()
t.Cleanup(func() {
cancel()
srv.Stop()
select {
case err := <-started:
if err != nil {
t.Errorf("srv.Start returned %v, want a clean stop", err)
}
case <-time.After(3 * time.Second):
t.Error("srv.Start did not return after Stop, so a channel leaked its goroutine")
}
})
if !listener.waitForState(t, 5*time.Second, sdnotify.ReadyState) {
t.Fatal("no READY=1 datagram within 5s of the Messaging Service serving")
}
}
+205
View File
@@ -0,0 +1,205 @@
// Package sdnotify implements systemd's sd_notify(3) protocol: one READY=1 when
// a process starts serving, and a WATCHDOG=1 heartbeat while the loop it stands
// for is demonstrably still making progress.
//
// The protocol is three environment variables and one datagram per state, so
// this package speaks it directly instead of taking a module dependency: the
// whole client is net.DialUnix plus a Write. Nothing here is fatal -- a process
// with no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without
// WatchdogSec) sends nothing and serves exactly as before, and a socket that
// cannot be written to is logged and ignored.
//
// It is cross-cutting per docs/architecture/organisation.md: any layer and any
// binary may import it, and it imports nothing of ours. It lives outside a
// composition package because a process that must announce itself cannot
// import another process's composition -- archied and archie-messaging are two
// processes that announce, and only one of them can own archied's boot
// (archie-core-g5g8).
//
// What stays with the caller is everything that is the caller's: which marker
// counts as progress, how long its loop may go without a pass, and the point
// at which it considers itself serving.
//
// The heartbeat is deliberately not a bare ticker. A ticker fires whether or
// not the loop it stands for is alive, so it would keep systemd satisfied while
// the event loop was wedged -- the exact failure the watchdog exists to catch.
// The sampler instead reads the caller's own progress marker through Progress
// and withholds the heartbeat once that marker is older than the cadence
// Cadence reports. While passes keep beginning, beats flow at half of
// WatchdogSec; a loop blocked inside a pass stops being reported as alive.
//
// Consuming the states is the unit's business: READY=1 needs Type=notify, and
// the heartbeat needs WatchdogSec.
package sdnotify
import (
"context"
"fmt"
"log/slog"
"net"
"os"
"strconv"
"time"
)
const (
// NotifySocketEnv names the socket a manager passes to a Type=notify unit.
NotifySocketEnv = "NOTIFY_SOCKET"
// WatchdogUsecEnv carries WatchdogSec in microseconds.
WatchdogUsecEnv = "WATCHDOG_USEC"
// WatchdogPIDEnv names the process whose beats the watchdog counts.
WatchdogPIDEnv = "WATCHDOG_PID"
// ReadyState and WatchdogState are the two sd_notify states a process
// announces: it has started serving, and it is still making progress.
ReadyState = "READY=1"
WatchdogState = "WATCHDOG=1"
// watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half
// of WatchdogSec, so one missed beat is tolerated and two are not.
watchdogHeartbeatDivisor = 2
)
// Notifier writes sd_notify states to the socket named in the environment. A
// notifier with no address -- the normal case away from systemd -- sends
// nothing and says nothing.
type Notifier struct {
addr string
log *slog.Logger
}
// New resolves the notify socket. The address is read from the environment
// systemd sets for the process, so systemd replaces the previous unit's socket
// with the new one across a restart.
func New(getenv func(string) string, log *slog.Logger) Notifier {
return Notifier{addr: getenv(NotifySocketEnv), log: log}
}
// Enabled reports whether a manager named a notify socket at all. It is the
// one fact a caller needs to decide whether a heartbeat is worth arming: with
// no socket every send is a no-op, and a sampler that cannot deliver a beat
// should not exist to withhold one.
func (n Notifier) Enabled() bool { return n.addr != "" }
// Send delivers one state line. A failure is logged, never returned: a
// notification is an observation about the process, and no observation is
// worth refusing to serve over.
func (n Notifier) Send(state string) {
if n.addr == "" {
return
}
if err := sendNotify(n.addr, state); err != nil {
n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err)
}
}
// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is
// one datagram per state, so each send dials afresh: a manager restart recreates
// its socket, and a connection cached across one would fail every send after it.
// A leading '@' names the Linux abstract namespace, which user managers use;
// net.UnixAddr takes that syntax verbatim.
func sendNotify(addr, state string) error {
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"})
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
if _, err := conn.Write([]byte(state)); err != nil {
return fmt.Errorf("write %q: %w", state, err)
}
return nil
}
// Ready sends READY=1 to the socket named in the process environment. It is
// the one call a composing package needs to declare itself serving, and it
// belongs at the moment the process already says so: the daemon's run loop
// (next to its own health surface), and the serve steps of archie-gateway,
// archie-state-store and archie-messaging, which have readiness of their own
// and would otherwise hang a Type=notify unit until TimeoutStartSec while
// perfectly healthy (archie-core-1174, archie-core-g5g8).
func Ready(log *slog.Logger) {
New(os.Getenv, log).Send(ReadyState)
}
// Heartbeat reads systemd's watchdog contract and returns the heartbeat
// cadence, which is half of WatchdogSec. ok is false when systemd asked for no
// watchdog, and also when it asked for one from a different process:
// WATCHDOG_PID names the process whose beats count, so a unit watching another
// process must not have this one beating on its behalf.
func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) {
raw := getenv(WatchdogUsecEnv)
if raw == "" {
return 0, false
}
if rawPID := getenv(WatchdogPIDEnv); rawPID != "" {
wantPID, err := strconv.Atoi(rawPID)
if err != nil || wantPID != pid {
log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process",
"watchdog_pid", rawPID, "pid", pid, "err", err)
return 0, false
}
}
watchdog, err := time.ParseDuration(raw + "us")
if err != nil || watchdog <= 0 {
log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err)
return 0, false
}
return watchdog / watchdogHeartbeatDivisor, true
}
// Watchdog samples the caller's progress marker and emits the watchdog
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
// not, because it is licensed by the marker rather than by the clock.
type Watchdog struct {
// Interval is the sampling cadence: half of systemd's WatchdogSec.
Interval time.Duration
// Cadence reports how long the loop may go between passes.
Cadence func() time.Duration
// Progress reports the start of the loop's most recent pass, or the zero
// time when no pass has begun yet.
Progress func() time.Time
// Beat emits one WATCHDOG=1.
Beat func()
Log *slog.Logger
}
// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
// only way out: this sampler never gives up on its own, because a sampler that
// stopped would leave systemd watching a process nothing reports for.
func (w Watchdog) Run(ctx context.Context) error {
armedAt := time.Now()
stalled := false
ticker := time.NewTicker(w.Interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
last := w.Progress()
if last.IsZero() {
// No pass has begun at all. Measure staleness from arming, so a
// loop that never starts polling is caught like one that stopped.
last = armedAt
}
cadence := w.Cadence()
switch age := time.Since(last); {
case age > cadence:
if !stalled {
w.Log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
"stale_for", age.String(), "loop_cadence", cadence.String())
stalled = true
}
default:
if stalled {
w.Log.Info("systemd watchdog: run loop resumed; heartbeats resumed",
"last_pass_age", age.String())
stalled = false
}
w.Beat()
}
}
}
+178
View File
@@ -0,0 +1,178 @@
package sdnotify
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"os"
"path/filepath"
"slices"
"sync"
"testing"
"time"
)
// notifyRecorder captures every record at every level. A warnings-only
// recorder would not do: these tests need "this path logged nothing at all" as
// positive evidence that a disabled notifier stayed silent, so the level has
// to be part of what is captured.
type notifyRecorder struct {
mu sync.Mutex
records []slog.Record
}
func (h *notifyRecorder) Enabled(context.Context, slog.Level) bool { return true }
func (h *notifyRecorder) Handle(_ context.Context, r slog.Record) error {
// The Record is the caller's to reuse once Handle returns, so keep a copy.
rec := r.Clone()
h.mu.Lock()
defer h.mu.Unlock()
h.records = append(h.records, rec)
return nil
}
func (h *notifyRecorder) WithAttrs([]slog.Attr) slog.Handler { return h }
func (h *notifyRecorder) WithGroup(string) slog.Handler { return h }
func (h *notifyRecorder) all() []slog.Record {
h.mu.Lock()
defer h.mu.Unlock()
return append([]slog.Record(nil), h.records...)
}
func (h *notifyRecorder) at(level slog.Level) []slog.Record {
h.mu.Lock()
defer h.mu.Unlock()
var out []slog.Record
for _, r := range h.records {
if r.Level == level {
out = append(out, r)
}
}
return out
}
// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket
// that collects the state lines a process sends. It is bound to the real
// NOTIFY_SOCKET for the duration of the test.
type notifyListener struct {
conn *net.UnixConn
}
// listenNotify binds a notify socket at addr and points NOTIFY_SOCKET at it.
func listenNotify(t *testing.T, addr string) *notifyListener {
t.Helper()
conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"})
if err != nil {
t.Fatalf("listen for notify datagrams at %q: %v", addr, err)
}
t.Cleanup(func() { _ = conn.Close() })
t.Setenv(NotifySocketEnv, addr)
return &notifyListener{conn: conn}
}
// states drains every state line received within window.
func (l *notifyListener) states(t *testing.T, window time.Duration) []string {
t.Helper()
deadline := time.Now().Add(window)
var got []string
buf := make([]byte, 1024)
for {
if err := l.conn.SetReadDeadline(deadline); err != nil {
t.Fatalf("set notify read deadline: %v", err)
}
n, _, err := l.conn.ReadFromUnix(buf)
if err != nil {
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
return got
}
t.Fatalf("read notify datagram: %v", err)
}
got = append(got, string(buf[:n]))
}
}
// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not
// restricted to filesystem paths, and a client that stripped the prefix would
// connect to a file that does not exist instead of the abstract namespace.
func TestNotifyReachesAbstractSocket(t *testing.T) {
listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid()))
Ready(slog.New(slog.DiscardHandler))
if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, ReadyState) {
t.Errorf("states received = %v, want %q among them", states, ReadyState)
}
}
// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the
// delivery contract: no socket means nothing is attempted and nothing is said,
// while a socket that cannot be written to is reported once and then ignored.
// Neither may ever be fatal.
func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) {
rec := &notifyRecorder{}
log := slog.New(rec)
New(func(string) string { return "" }, log).Send(ReadyState)
if got := rec.all(); len(got) != 0 {
t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got)
}
missing := filepath.Join(t.TempDir(), "absent", "n.sock")
New(func(string) string { return missing }, log).Send(ReadyState)
warns := rec.at(slog.LevelWarn)
if len(warns) != 1 {
t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns))
}
if errs := rec.at(slog.LevelError); len(errs) != 0 {
t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs))
}
}
// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes
// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and
// WATCHDOG_PID names the process whose beats count.
func TestWatchdogHeartbeatInterval(t *testing.T) {
const self = 42
tests := []struct {
name string
usec string
pidEnv string
want time.Duration
wantOK bool
}{
{name: "no watchdog requested", wantOK: false},
{name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true},
{name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true},
{name: "unparseable", usec: "soon", wantOK: false},
{name: "zero", usec: "0", wantOK: false},
{name: "negative", usec: "-1", wantOK: false},
{name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true},
{name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false},
{name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
getenv := func(key string) string {
switch key {
case WatchdogUsecEnv:
return tt.usec
case WatchdogPIDEnv:
return tt.pidEnv
default:
return ""
}
}
got, ok := Heartbeat(getenv, self, slog.New(slog.DiscardHandler))
if ok != tt.wantOK || got != tt.want {
t.Errorf("Heartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v",
tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK)
}
})
}
}