feat(archied): report systemd readiness and loop-driven watchdog heartbeats
archied now speaks sd_notify(3): READY=1 at the moment the process declares itself serving, and WATCHDOG=1 heartbeats while the run loop's own progress marker is fresh. The heartbeat is licensed by that marker (daemon.LastPollAt, stamped at the start of every poll pass) rather than by an independent ticker, so a loop wedged inside a pass stops being reported as alive and trips the unit's watchdog. The protocol is three environment variables and one datagram per state, so it is spoken directly with net.DialUnix instead of through a new dependency. Without NOTIFY_SOCKET nothing is sent, nothing is logged, and startup is unaffected.
This commit is contained in:
3 files changed
+615
No files matched your search
@@ -1566,6 +1566,13 @@ func (b *boot) runLoop(ctx context.Context, once bool) error {
|
||||
// Boot is done: the liveness surface stops answering 503, which is what
|
||||
// the update watchdog is waiting to see after a restart.
|
||||
b.health.markServing()
|
||||
// systemd's READY=1 asserts the same fact this line has just recorded, so
|
||||
// it is sent here rather than from a second notion of "started". Not
|
||||
// earlier: the stores, NATS, gateways and tool providers are all up by now.
|
||||
// Not later, after the first pass: a pass polls every repo and drains the
|
||||
// queue, and a unit whose TimeoutStartSec expires mid-pass would be
|
||||
// restarted while the daemon was working.
|
||||
b.announceReady()
|
||||
if once {
|
||||
b.d.Cycle(ctx)
|
||||
return nil
|
||||
@@ -1575,6 +1582,9 @@ func (b *boot) runLoop(ctx context.Context, once bool) error {
|
||||
// a --once cycle exits immediately, so a drain that arrives mid-cycle is
|
||||
// out of scope and the monitor is not started.
|
||||
b.startDrainMonitor(ctx)
|
||||
// Armed here, not at boot: the loop it reports on starts on the next line,
|
||||
// and a --once invocation has no loop to watchdog.
|
||||
b.startWatchdog(ctx)
|
||||
if err := b.d.Run(ctx); err != nil && ctx.Err() == nil {
|
||||
b.log.Error("daemon exited", "err", err)
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
// sd_notify.go implements systemd's sd_notify(3) protocol for archied: one
|
||||
// READY=1 when the run loop starts serving, and a WATCHDOG=1 heartbeat while
|
||||
// the run loop is demonstrably still making progress.
|
||||
//
|
||||
// The protocol is three environment variables and one datagram per state, so
|
||||
// this file speaks it directly instead of taking a module dependency: the whole
|
||||
// client is net.DialUnix plus a Write. Nothing here is fatal -- a process with
|
||||
// no NOTIFY_SOCKET (a plain terminal, Docker, a user unit without WatchdogSec)
|
||||
// sends nothing and serves exactly as before, and a socket that cannot be
|
||||
// written to is logged and ignored.
|
||||
//
|
||||
// The heartbeat is deliberately not a bare ticker. A ticker fires whether or
|
||||
// not the loop it stands for is alive, so it would keep systemd satisfied while
|
||||
// the event loop was wedged -- the exact failure the watchdog exists to catch.
|
||||
// The sampler instead reads the daemon's own progress marker
|
||||
// (daemon.LastPollAt, stamped by the poll loop at the start of every pass) and
|
||||
// withholds the heartbeat once that marker is older than the loop's own poll
|
||||
// cadence. While passes keep beginning, beats flow at half of WatchdogSec; a
|
||||
// loop blocked inside a pass stops being reported as alive.
|
||||
//
|
||||
// Consuming the states is the unit's business: READY=1 needs Type=notify, and
|
||||
// the heartbeat needs WatchdogSec.
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// notifySocketEnv names the socket a manager passes to a Type=notify unit.
|
||||
notifySocketEnv = "NOTIFY_SOCKET"
|
||||
// watchdogUsecEnv carries WatchdogSec in microseconds.
|
||||
watchdogUsecEnv = "WATCHDOG_USEC"
|
||||
// watchdogPIDEnv names the process whose beats the watchdog counts.
|
||||
watchdogPIDEnv = "WATCHDOG_PID"
|
||||
|
||||
// readyState and watchdogState are the two sd_notify states archied sends.
|
||||
readyState = "READY=1"
|
||||
watchdogState = "WATCHDOG=1"
|
||||
|
||||
// watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half
|
||||
// of WatchdogSec, so one missed beat is tolerated and two are not.
|
||||
watchdogHeartbeatDivisor = 2
|
||||
)
|
||||
|
||||
// notifier writes sd_notify states to the socket named in the environment. A
|
||||
// notifier with no address -- the normal case away from systemd -- sends
|
||||
// nothing and says nothing.
|
||||
type notifier struct {
|
||||
addr string
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// newNotifier resolves the notify socket. The address is read from the
|
||||
// environment systemd sets for the process, so systemd replaces the previous
|
||||
// unit's socket with the new one across a restart.
|
||||
func newNotifier(getenv func(string) string, log *slog.Logger) notifier {
|
||||
return notifier{addr: getenv(notifySocketEnv), log: log}
|
||||
}
|
||||
|
||||
// send delivers one state line. A failure is logged, never returned: a
|
||||
// notification is an observation about the daemon, and no observation is worth
|
||||
// refusing to serve over.
|
||||
func (n notifier) send(state string) {
|
||||
if n.addr == "" {
|
||||
return
|
||||
}
|
||||
if err := sendNotify(n.addr, state); err != nil {
|
||||
n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// sendNotify writes one state datagram to a unixgram socket. sd_notify(3) is
|
||||
// one datagram per state, so each send dials afresh: a manager restart recreates
|
||||
// its socket, and a connection cached across one would fail every send after it.
|
||||
// A leading '@' names the Linux abstract namespace, which user managers use;
|
||||
// net.UnixAddr takes that syntax verbatim.
|
||||
func sendNotify(addr, state string) error {
|
||||
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = conn.Close() }()
|
||||
if _, err := conn.Write([]byte(state)); err != nil {
|
||||
return fmt.Errorf("write %q: %w", state, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// announceReady tells systemd the daemon has finished starting. It is called at
|
||||
// the moment the process declares itself serving (runLoop, next to
|
||||
// healthSurface.markServing), which is the same fact READY=1 asserts: boot is
|
||||
// over and the run loop is about to take over.
|
||||
func (b *boot) announceReady() {
|
||||
newNotifier(os.Getenv, b.log).send(readyState)
|
||||
}
|
||||
|
||||
// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does
|
||||
// nothing unless a manager named a notify socket and asked for a watchdog
|
||||
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
|
||||
func (b *boot) startWatchdog(ctx context.Context) {
|
||||
n := newNotifier(os.Getenv, b.log)
|
||||
if n.addr == "" {
|
||||
return
|
||||
}
|
||||
interval, ok := watchdogHeartbeat(os.Getenv, os.Getpid(), b.log)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
sampler := loopWatchdog{
|
||||
interval: interval,
|
||||
cadence: b.loopCadence,
|
||||
progress: b.d.LastPollAt,
|
||||
beat: func() { n.send(watchdogState) },
|
||||
log: b.log,
|
||||
}
|
||||
b.log.Info("systemd watchdog armed",
|
||||
"heartbeat", interval.String(),
|
||||
"loop_cadence", b.loopCadence().String())
|
||||
|
||||
go func() {
|
||||
if err := sampler.run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
b.log.Error("systemd watchdog stopped", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// loopCadence is the longest interval at which the run loop may go without
|
||||
// beginning a fresh poll pass, and therefore how stale the progress marker may
|
||||
// be before the loop counts as stalled.
|
||||
//
|
||||
// It is the root poll interval, or the slowest configured identity in
|
||||
// multi-identity mode: each identity polls its own repos, identity poll
|
||||
// intervals are frozen at boot, and the marker is stamped by whichever identity
|
||||
// polled most recently -- so the worst case between stamps is the slowest
|
||||
// identity, not the fastest. The root value is re-read per sample, so a
|
||||
// reloaded poll_interval is honoured.
|
||||
func (b *boot) loopCadence() time.Duration {
|
||||
cadence := b.cfgHolder.Get().PollInterval.Std()
|
||||
for _, id := range b.d.Identities {
|
||||
if override := id.Cfg.PollInterval.Std(); override > cadence {
|
||||
cadence = override
|
||||
}
|
||||
}
|
||||
return cadence
|
||||
}
|
||||
|
||||
// watchdogHeartbeat reads systemd's watchdog contract and returns the heartbeat
|
||||
// cadence, which is half of WatchdogSec. ok is false when systemd asked for no
|
||||
// watchdog, and also when it asked for one from a different process:
|
||||
// WATCHDOG_PID names the process whose beats count, so a unit watching another
|
||||
// process must not have this one beating on its behalf.
|
||||
func watchdogHeartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) {
|
||||
raw := getenv(watchdogUsecEnv)
|
||||
if raw == "" {
|
||||
return 0, false
|
||||
}
|
||||
if rawPID := getenv(watchdogPIDEnv); rawPID != "" {
|
||||
wantPID, err := strconv.Atoi(rawPID)
|
||||
if err != nil || wantPID != pid {
|
||||
log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process",
|
||||
"watchdog_pid", rawPID, "pid", pid, "err", err)
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
watchdog, err := time.ParseDuration(raw + "us")
|
||||
if err != nil || watchdog <= 0 {
|
||||
log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err)
|
||||
return 0, false
|
||||
}
|
||||
return watchdog / watchdogHeartbeatDivisor, true
|
||||
}
|
||||
|
||||
// loopWatchdog samples the run loop's progress marker and emits the watchdog
|
||||
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
|
||||
// not, because it is licensed by the marker rather than by the clock.
|
||||
type loopWatchdog struct {
|
||||
// interval is the sampling cadence: half of systemd's WatchdogSec.
|
||||
interval time.Duration
|
||||
// cadence reports how long the loop may go between poll passes.
|
||||
cadence func() time.Duration
|
||||
// progress reports the start of the loop's most recent poll pass, or the
|
||||
// zero time when no pass has begun yet.
|
||||
progress func() time.Time
|
||||
// beat emits one WATCHDOG=1.
|
||||
beat func()
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
|
||||
// only way out: this sampler never gives up on its own, because a sampler that
|
||||
// stopped would leave systemd watching a process nothing reports for.
|
||||
func (w loopWatchdog) run(ctx context.Context) error {
|
||||
armedAt := time.Now()
|
||||
stalled := false
|
||||
ticker := time.NewTicker(w.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
|
||||
last := w.progress()
|
||||
if last.IsZero() {
|
||||
// No pass has begun at all. Measure staleness from arming, so a
|
||||
// loop that never starts polling is caught like one that stopped.
|
||||
last = armedAt
|
||||
}
|
||||
cadence := w.cadence()
|
||||
switch age := time.Since(last); {
|
||||
case age > cadence:
|
||||
if !stalled {
|
||||
w.log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
|
||||
"stale_for", age.String(), "loop_cadence", cadence.String())
|
||||
stalled = true
|
||||
}
|
||||
default:
|
||||
if stalled {
|
||||
w.log.Info("systemd watchdog: run loop resumed; heartbeats resumed",
|
||||
"last_pass_age", age.String())
|
||||
stalled = false
|
||||
}
|
||||
w.beat()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/store"
|
||||
)
|
||||
|
||||
// notifyRecorder captures every record at every level. The other recording
|
||||
// handler in this package keeps warnings only; these tests need "this path
|
||||
// logged nothing at all" as positive evidence that a disabled notifier stayed
|
||||
// silent, so the level has to be part of what is captured.
|
||||
type notifyRecorder struct {
|
||||
mu sync.Mutex
|
||||
records []slog.Record
|
||||
}
|
||||
|
||||
func (h *notifyRecorder) Enabled(context.Context, slog.Level) bool { return true }
|
||||
|
||||
func (h *notifyRecorder) Handle(_ context.Context, r slog.Record) error {
|
||||
// The Record is the caller's to reuse once Handle returns, so keep a copy.
|
||||
rec := r.Clone()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.records = append(h.records, rec)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *notifyRecorder) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
func (h *notifyRecorder) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
func (h *notifyRecorder) all() []slog.Record {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return append([]slog.Record(nil), h.records...)
|
||||
}
|
||||
|
||||
func (h *notifyRecorder) at(level slog.Level) []slog.Record {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []slog.Record
|
||||
for _, r := range h.records {
|
||||
if r.Level == level {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notifyListener is a stand-in for systemd's NOTIFY_SOCKET: a unixgram socket
|
||||
// that collects the state lines the daemon sends. It is bound to the real
|
||||
// NOTIFY_SOCKET for the duration of the test.
|
||||
type notifyListener struct {
|
||||
conn *net.UnixConn
|
||||
}
|
||||
|
||||
func newNotifyListener(t *testing.T) *notifyListener {
|
||||
t.Helper()
|
||||
// A unix socket path is capped near 108 bytes; the short file name keeps the
|
||||
// path inside t.TempDir() under that for any test name.
|
||||
return listenNotify(t, filepath.Join(t.TempDir(), "n.sock"))
|
||||
}
|
||||
|
||||
// listenNotify binds a notify socket at addr and points NOTIFY_SOCKET at it.
|
||||
func listenNotify(t *testing.T, addr string) *notifyListener {
|
||||
t.Helper()
|
||||
conn, err := net.ListenUnixgram("unixgram", &net.UnixAddr{Name: addr, Net: "unixgram"})
|
||||
if err != nil {
|
||||
t.Fatalf("listen for notify datagrams at %q: %v", addr, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
t.Setenv(notifySocketEnv, addr)
|
||||
return ¬ifyListener{conn: conn}
|
||||
}
|
||||
|
||||
// states drains every state line received within window.
|
||||
func (l *notifyListener) states(t *testing.T, window time.Duration) []string {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(window)
|
||||
var got []string
|
||||
buf := make([]byte, 1024)
|
||||
for {
|
||||
if err := l.conn.SetReadDeadline(deadline); err != nil {
|
||||
t.Fatalf("set notify read deadline: %v", err)
|
||||
}
|
||||
n, _, err := l.conn.ReadFromUnix(buf)
|
||||
if err != nil {
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return got
|
||||
}
|
||||
t.Fatalf("read notify datagram: %v", err)
|
||||
}
|
||||
got = append(got, string(buf[:n]))
|
||||
}
|
||||
}
|
||||
|
||||
// countState counts the datagrams carrying want among the received states.
|
||||
func countState(states []string, want string) int {
|
||||
n := 0
|
||||
for _, state := range states {
|
||||
if state == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// newNotifyTestBoot builds the composition the notify path reads: a boot whose
|
||||
// config holder supplies the run loop's poll cadence, and a real daemon whose
|
||||
// own poll loop stamps the progress marker the heartbeat sampler samples.
|
||||
func newNotifyTestBoot(t *testing.T, cadence time.Duration) (*boot, *daemon.Daemon) {
|
||||
t.Helper()
|
||||
cfg := config.Config{PollInterval: config.Duration(cadence)}
|
||||
holder := config.NewHolder(cfg)
|
||||
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "tasks.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open task store: %v", err)
|
||||
}
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
d := &daemon.Daemon{Cfg: holder, Store: st, Log: log}
|
||||
return &boot{cfg: cfg, cfgHolder: holder, d: d, log: log}, d
|
||||
}
|
||||
|
||||
// pollPasses drives the daemon's poll passes the way Run does, one Cycle per
|
||||
// tick, while on is set. The progress marker is stamped by the daemon inside
|
||||
// Cycle, never by the test: what the sampler samples is the loop's own
|
||||
// evidence, not a value the test wrote.
|
||||
func pollPasses(ctx context.Context, d *daemon.Daemon, on *atomic.Bool, every time.Duration) {
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if on.Load() {
|
||||
d.Cycle(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyReachesAbstractSocket pins the '@' address form. systemd is not
|
||||
// restricted to filesystem paths, and a client that stripped the prefix would
|
||||
// connect to a file that does not exist instead of the abstract namespace.
|
||||
func TestNotifyReachesAbstractSocket(t *testing.T) {
|
||||
listener := listenNotify(t, fmt.Sprintf("@archie-notify-test-%d", os.Getpid()))
|
||||
|
||||
(&boot{log: slog.New(slog.DiscardHandler)}).announceReady()
|
||||
|
||||
if states := listener.states(t, 300*time.Millisecond); !slices.Contains(states, readyState) {
|
||||
t.Errorf("states received = %v, want %q among them", states, readyState)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifierDisabledIsSilentAndUnusableSocketWarns pins the two ends of the
|
||||
// delivery contract: no socket means nothing is attempted and nothing is said,
|
||||
// while a socket that cannot be written to is reported once and then ignored.
|
||||
// Neither may ever be fatal.
|
||||
func TestNotifierDisabledIsSilentAndUnusableSocketWarns(t *testing.T) {
|
||||
rec := ¬ifyRecorder{}
|
||||
log := slog.New(rec)
|
||||
|
||||
newNotifier(func(string) string { return "" }, log).send(readyState)
|
||||
if got := rec.all(); len(got) != 0 {
|
||||
t.Errorf("a notifier with no socket logged %d records, want none: %v", len(got), got)
|
||||
}
|
||||
|
||||
missing := filepath.Join(t.TempDir(), "absent", "n.sock")
|
||||
newNotifier(func(string) string { return missing }, log).send(readyState)
|
||||
warns := rec.at(slog.LevelWarn)
|
||||
if len(warns) != 1 {
|
||||
t.Fatalf("sends to an unusable socket logged %d warnings, want exactly 1", len(warns))
|
||||
}
|
||||
if errs := rec.at(slog.LevelError); len(errs) != 0 {
|
||||
t.Errorf("sends to an unusable socket logged %d errors, want none", len(errs))
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogHeartbeatInterval pins the interval convention: systemd passes
|
||||
// WatchdogSec as WATCHDOG_USEC and sd_notify(3) heartbeats at half of it, and
|
||||
// WATCHDOG_PID names the process whose beats count.
|
||||
func TestWatchdogHeartbeatInterval(t *testing.T) {
|
||||
const self = 42
|
||||
tests := []struct {
|
||||
name string
|
||||
usec string
|
||||
pidEnv string
|
||||
want time.Duration
|
||||
wantOK bool
|
||||
}{
|
||||
{name: "no watchdog requested", wantOK: false},
|
||||
{name: "half of a 30s watchdog", usec: "30000000", want: 15 * time.Second, wantOK: true},
|
||||
{name: "sub-second watchdog", usec: "20000", want: 10 * time.Millisecond, wantOK: true},
|
||||
{name: "unparseable", usec: "soon", wantOK: false},
|
||||
{name: "zero", usec: "0", wantOK: false},
|
||||
{name: "negative", usec: "-1", wantOK: false},
|
||||
{name: "watchdog pid is this process", usec: "20000", pidEnv: "42", want: 10 * time.Millisecond, wantOK: true},
|
||||
{name: "watchdog pid is another process", usec: "20000", pidEnv: "7", wantOK: false},
|
||||
{name: "watchdog pid is unreadable", usec: "20000", pidEnv: "seven", wantOK: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
getenv := func(key string) string {
|
||||
switch key {
|
||||
case watchdogUsecEnv:
|
||||
return tt.usec
|
||||
case watchdogPIDEnv:
|
||||
return tt.pidEnv
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
got, ok := watchdogHeartbeat(getenv, self, slog.New(slog.DiscardHandler))
|
||||
if ok != tt.wantOK || got != tt.want {
|
||||
t.Errorf("watchdogHeartbeat(usec=%q, pid=%q) = %v, %v; want %v, %v",
|
||||
tt.usec, tt.pidEnv, got, ok, tt.want, tt.wantOK)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyReadyAndWatchdogHeartbeat is the run-loop path end to end: a real
|
||||
// NOTIFY_SOCKET, the daemon's own boot path announcing READY, and the run
|
||||
// loop's own progress marker producing WATCHDOG heartbeats.
|
||||
func TestNotifyReadyAndWatchdogHeartbeat(t *testing.T) {
|
||||
listener := newNotifyListener(t)
|
||||
t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
|
||||
|
||||
b, d := newNotifyTestBoot(t, 50*time.Millisecond)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
var polling atomic.Bool
|
||||
polling.Store(true)
|
||||
go pollPasses(ctx, d, &polling, 5*time.Millisecond)
|
||||
|
||||
b.announceReady()
|
||||
b.startWatchdog(ctx)
|
||||
|
||||
states := listener.states(t, 300*time.Millisecond)
|
||||
if !slices.Contains(states, readyState) {
|
||||
t.Errorf("states received = %v, want %q among them", states, readyState)
|
||||
}
|
||||
if countState(states, watchdogState) == 0 {
|
||||
t.Errorf("states received = %v, want at least one %q among them", states, watchdogState)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchdogWithholdsHeartbeatWhenRunLoopStalls is the semantic that makes
|
||||
// the heartbeat worth having: the beats stop once the loop stops making
|
||||
// progress, so a wedged event loop trips systemd's watchdog instead of being
|
||||
// kept alive by an independent ticker.
|
||||
func TestWatchdogWithholdsHeartbeatWhenRunLoopStalls(t *testing.T) {
|
||||
listener := newNotifyListener(t)
|
||||
t.Setenv(watchdogUsecEnv, "20000") // 20ms watchdog: heartbeat every 10ms
|
||||
|
||||
const cadence = 50 * time.Millisecond
|
||||
b, d := newNotifyTestBoot(t, cadence)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
var polling atomic.Bool
|
||||
polling.Store(true)
|
||||
go pollPasses(ctx, d, &polling, 5*time.Millisecond)
|
||||
|
||||
b.startWatchdog(ctx)
|
||||
if beats := countState(listener.states(t, 200*time.Millisecond), watchdogState); beats == 0 {
|
||||
t.Fatal("no WATCHDOG=1 within 200ms while the run loop was polling")
|
||||
}
|
||||
|
||||
// From here the loop is inside a pass that never returns: nothing stamps
|
||||
// the progress marker again.
|
||||
polling.Store(false)
|
||||
|
||||
// Discard the beats the last stamp still licenses. The marker counts as
|
||||
// stale only once it is older than the loop's own cadence (50ms), and
|
||||
// samples are 10ms apart, so 150ms covers every remaining beat with margin.
|
||||
listener.states(t, 150*time.Millisecond)
|
||||
|
||||
if late := listener.states(t, 300*time.Millisecond); len(late) != 0 {
|
||||
t.Errorf("received %v after the run loop stalled, want no heartbeats", late)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifyUnsetIsNoOp pins the deployment contract: with no NOTIFY_SOCKET the
|
||||
// process behaves exactly as it did before sd_notify existed. Docker, a plain
|
||||
// terminal, and a systemd unit with no WatchdogSec all take this path, and none
|
||||
// of them may see a datagram, a warning, or a failed start.
|
||||
func TestNotifyUnsetIsNoOp(t *testing.T) {
|
||||
t.Setenv(watchdogUsecEnv, "20000")
|
||||
// t.Setenv cannot unset a variable: register the restoration first, then
|
||||
// remove it for real so "unset" means unset.
|
||||
t.Setenv(notifySocketEnv, "")
|
||||
if err := os.Unsetenv(notifySocketEnv); err != nil {
|
||||
t.Fatalf("unset %s: %v", notifySocketEnv, err)
|
||||
}
|
||||
|
||||
rec := ¬ifyRecorder{}
|
||||
b, _ := newNotifyTestBoot(t, 50*time.Millisecond)
|
||||
b.log = slog.New(rec)
|
||||
|
||||
b.announceReady()
|
||||
b.startWatchdog(t.Context())
|
||||
|
||||
if got := rec.all(); len(got) != 0 {
|
||||
t.Errorf("with NOTIFY_SOCKET unset the notify path logged %d records, want none: %v", len(got), got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotifySocketFailureIsNotFatal pins the other half of the deployment
|
||||
// contract: a NOTIFY_SOCKET that cannot be written to is logged and ignored.
|
||||
// A daemon that refused to serve because systemd's socket was unavailable
|
||||
// would be a regression on every deployment that has no systemd at all.
|
||||
func TestNotifySocketFailureIsNotFatal(t *testing.T) {
|
||||
missing := filepath.Join(t.TempDir(), "absent", "n.sock")
|
||||
t.Setenv(notifySocketEnv, missing)
|
||||
t.Setenv(watchdogUsecEnv, "20000")
|
||||
|
||||
rec := ¬ifyRecorder{}
|
||||
b, d := newNotifyTestBoot(t, 50*time.Millisecond)
|
||||
b.log = slog.New(rec)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
|
||||
var polling atomic.Bool
|
||||
polling.Store(true)
|
||||
go pollPasses(ctx, d, &polling, 5*time.Millisecond)
|
||||
|
||||
b.announceReady()
|
||||
b.startWatchdog(ctx)
|
||||
|
||||
if warns := rec.at(slog.LevelWarn); len(warns) != 1 {
|
||||
t.Errorf("announceReady against an unusable socket logged %d warnings, want exactly 1", len(warns))
|
||||
}
|
||||
|
||||
// The sampler keeps trying rather than dying on the first failure, so more
|
||||
// warnings arrive while the loop progresses.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if len(rec.at(slog.LevelWarn)) > 1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if warns := rec.at(slog.LevelWarn); len(warns) < 2 {
|
||||
t.Errorf("the watchdog logged %d warnings, want the heartbeat still being attempted", len(warns))
|
||||
}
|
||||
if errs := rec.at(slog.LevelError); len(errs) != 0 {
|
||||
t.Errorf("an unusable notify socket logged %d errors, want none: %v", len(errs), errs)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user