From f44db549482ff479729e731494577df08042e32b Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 9 Nov 2026 14:36:08 +1100 Subject: [PATCH] fix(archied): the watchdog heartbeat proves liveness and a stalled run loop reports degraded readiness (#1840) State Store calls the daemon makes without a deadline now get one. --- internal/app/archied/bootstrap.go | 2 +- internal/app/archied/readiness.go | 35 ++++++++++++++++++ internal/app/archied/sd_notify.go | 23 ++++-------- internal/app/servicekit/statestore.go | 6 ++- internal/infrastructure/staterpc/dial.go | 14 +++++++ internal/sdnotify/sdnotify.go | 47 ++++-------------------- 6 files changed, 68 insertions(+), 59 deletions(-) diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index c6b260b21..43ad82a85 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -289,7 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error { if target == "" { return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md ยง12 step 7)") } - client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg) + client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline)) if err != nil { b.log.ErrorContext(ctx, "state store adapter", "err", err) return err diff --git a/internal/app/archied/readiness.go b/internal/app/archied/readiness.go index 116130ad0..22146f6e9 100644 --- a/internal/app/archied/readiness.go +++ b/internal/app/archied/readiness.go @@ -34,6 +34,12 @@ func (b *boot) setupReadinessProbes() { _, err := b.stateStore.StatusCounts(ctx) return err }), + runLoopProbe{stalest: func() (string, time.Time) { + if b.d == nil { + return "", time.Time{} + } + return b.d.StalestLoop() + }, cadence: b.loopCadence}, readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate), readiness.NewDiskProbeTargets(diskProbeTargets(cfg)), // The daemon no longer runs a chat channel, so it cannot observe one's @@ -100,3 +106,32 @@ func (unscopedBroker) Name() string { return "nats_credential_scope" } func (unscopedBroker) Check(context.Context) health.Result { return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"} } + +// stateCallDeadline bounds each State Store call the daemon makes without a +// deadline of its own. +const stateCallDeadline = 20 * time.Second + +// runLoopStallFactor is how many poll intervals a loop may go without +// beginning a pass before it is reported degraded. +const runLoopStallFactor = 3 + +// runLoopProbe reports a run loop that has stopped beginning passes, for +// instance blocked on a slow State Store. +type runLoopProbe struct { + stalest func() (string, time.Time) + cadence func() time.Duration +} + +func (runLoopProbe) Name() string { return "run_loop" } + +func (p runLoopProbe) Check(context.Context) health.Result { + loop, last := p.stalest() + if last.IsZero() { + return health.Result{Status: health.StatusOK} + } + limit := max(p.cadence()*runLoopStallFactor, time.Minute) + if age := time.Since(last); age > limit { + return health.Result{Status: health.StatusDegraded, Detail: loop + " loop has not begun a pass for " + age.Round(time.Second).String()} + } + return health.Result{Status: health.StatusOK} +} diff --git a/internal/app/archied/sd_notify.go b/internal/app/archied/sd_notify.go index f5bac6b5a..8e8488204 100644 --- a/internal/app/archied/sd_notify.go +++ b/internal/app/archied/sd_notify.go @@ -1,14 +1,12 @@ // sd_notify.go wires the daemon into internal/sdnotify's implementation of // systemd's sd_notify(3) protocol: one READY=1 when the run loop starts -// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still -// making progress. +// serving, and a WATCHDOG=1 heartbeat while the process is alive. // // The protocol client and the heartbeat sampler live in internal/sdnotify // because archie-messaging, a separate process that cannot import this // package, is the next process that must be able to announce itself. -// What stays here is what is the daemon's own: which -// marker counts as progress (daemon.StalestLoop) and how long a loop may go -// without beginning a poll pass. +// A run loop that stalls on a slow peer is reported by the run_loop readiness +// probe, never by withholding the heartbeat. // // Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal, // Docker, a user unit without WatchdogSec) sends nothing and serves exactly as @@ -37,7 +35,7 @@ func (b *boot) announceReady() { sdnotify.Ready(b.log) } -// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does +// startWatchdog arms the heartbeat. It does // nothing unless a manager named a notify socket and asked for a watchdog // (WATCHDOG_USEC), so every deployment without systemd is unaffected. func (b *boot) startWatchdog(ctx context.Context) { @@ -52,14 +50,10 @@ func (b *boot) startWatchdog(ctx context.Context) { sampler := sdnotify.Watchdog{ Interval: interval, - Cadence: b.loopCadence, - Progress: b.d.StalestLoop, Beat: func() { n.Send(sdnotify.WatchdogState) }, - Log: b.log, } b.log.InfoContext(ctx, "systemd watchdog armed", - "heartbeat", interval.String(), - "loop_cadence", b.loopCadence().String()) + "heartbeat", interval.String()) go func() { if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { @@ -68,11 +62,8 @@ func (b *boot) startWatchdog(ctx context.Context) { }() } -// loopCadence is the longest interval at which the run loop may go without -// beginning a fresh poll pass, and therefore how stale the progress marker may -// be before the loop counts as stalled. -// -// The shared interval is re-read so live polling changes are honoured. +// loopCadence is the interval at which the run loop begins a poll pass. The +// shared interval is re-read so live polling changes are honoured. func (b *boot) loopCadence() time.Duration { return b.cfgHolder.Get().PollInterval.Std() } diff --git a/internal/app/servicekit/statestore.go b/internal/app/servicekit/statestore.go index 8cc3754de..2e46228e5 100644 --- a/internal/app/servicekit/statestore.go +++ b/internal/app/servicekit/statestore.go @@ -1,6 +1,8 @@ package servicekit import ( + "google.golang.org/grpc" + "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" @@ -15,7 +17,7 @@ func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) { // StateStoreClient dials the State Store gRPC service as the caller service. // Calls wait for the State Store rather than fail while it is down, so boot // order between services does not matter. -func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) { +func StateStoreClient(caller string, cfg config.Config, options ...grpc.DialOption) (*staterpc.Client, func(), error) { target, err := cfg.Services.RequireTarget(config.ServiceNameState) if err != nil { return nil, nil, err @@ -24,5 +26,5 @@ func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func( if err != nil { return nil, nil, err } - return staterpc.Dial(target, caller, id, staterpc.WaitForPeer) + return staterpc.Dial(target, caller, id, append([]grpc.DialOption{staterpc.WaitForPeer}, options...)...) } diff --git a/internal/infrastructure/staterpc/dial.go b/internal/infrastructure/staterpc/dial.go index daba94d07..10450ff62 100644 --- a/internal/infrastructure/staterpc/dial.go +++ b/internal/infrastructure/staterpc/dial.go @@ -1,6 +1,7 @@ package staterpc import ( + "context" "fmt" "strings" "time" @@ -93,3 +94,16 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy { // instead of failing while it is down or restarting. A call that must not // wait carries its own deadline. var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true)) + +// WithCallDeadline bounds every unary call that carries no deadline of its +// own, so a slow State Store fails the call instead of parking its caller. +func WithCallDeadline(d time.Duration) grpc.DialOption { + return grpc.WithChainUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, d) + defer cancel() + } + return invoker(ctx, method, req, reply, cc, opts...) + }) +} diff --git a/internal/sdnotify/sdnotify.go b/internal/sdnotify/sdnotify.go index 78a234f16..ca019d217 100644 --- a/internal/sdnotify/sdnotify.go +++ b/internal/sdnotify/sdnotify.go @@ -1,6 +1,5 @@ // Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages. -// Heartbeats stop when the caller's progress marker goes stale. Without -// NOTIFY_SOCKET nothing is sent. +// Without NOTIFY_SOCKET nothing is sent. package sdnotify import ( @@ -106,58 +105,26 @@ func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Dura return watchdog / watchdogHeartbeatDivisor, true } -// Watchdog samples the caller's progress marker and emits the watchdog -// heartbeat only while that marker is fresh. Sampling is periodic; the beat is -// not, because it is licensed by the marker rather than by the clock. +// Watchdog emits the watchdog heartbeat on every tick for as long as the +// process can run a goroutine. It proves the process is alive, not that any +// one piece of work finished: a peer that answers slowly must degrade +// readiness, not get the process killed. type Watchdog struct { - // Interval is the sampling cadence: half of systemd's WatchdogSec. + // Interval is the beat cadence: half of systemd's WatchdogSec. Interval time.Duration - // Cadence reports how long the loop may go between passes. - Cadence func() time.Duration - // Progress reports the stalest loop and the start of its most recent - // pass, or the zero time when no pass has begun yet. - Progress func() (string, time.Time) // Beat emits one WATCHDOG=1. Beat func() - Log *slog.Logger } -// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the -// only way out: this sampler never gives up on its own, because a sampler that -// stopped would leave systemd watching a process nothing reports for. +// Run beats until ctx ends and returns ctx.Err(). func (w Watchdog) Run(ctx context.Context) error { - armedAt := time.Now() - stalled := false ticker := time.NewTicker(w.Interval) defer ticker.Stop() - for { select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: - } - - loop, last := w.Progress() - if last.IsZero() { - // No pass has begun at all. Measure staleness from arming, so a - // loop that never starts polling is caught like one that stopped. - last = armedAt - } - cadence := w.Cadence() - switch age := time.Since(last); { - case age > cadence: - if !stalled { - w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does", - "loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String()) - stalled = true - } - default: - if stalled { - w.Log.InfoContext(ctx, "systemd watchdog: run loop resumed; heartbeats resumed", - "last_pass_age", age.String()) - stalled = false - } w.Beat() } }