fix(archied): the watchdog heartbeat proves liveness and a stalled run loop reports degraded readiness (#1840)

State Store calls the daemon makes without a deadline now get one.
This commit is contained in:
sam committed 2026-11-09 14:36:08 +11:00
1 parent 44b52c54c0
commit f44db54948
6 files changed
+68 -59

No files matched your search

+1 -1
View File
@@ -289,7 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
if target == "" {
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
}
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg)
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline))
if err != nil {
b.log.ErrorContext(ctx, "state store adapter", "err", err)
return err
+35
View File
@@ -34,6 +34,12 @@ func (b *boot) setupReadinessProbes() {
_, err := b.stateStore.StatusCounts(ctx)
return err
}),
runLoopProbe{stalest: func() (string, time.Time) {
if b.d == nil {
return "", time.Time{}
}
return b.d.StalestLoop()
}, cadence: b.loopCadence},
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
// The daemon no longer runs a chat channel, so it cannot observe one's
@@ -100,3 +106,32 @@ func (unscopedBroker) Name() string { return "nats_credential_scope" }
func (unscopedBroker) Check(context.Context) health.Result {
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
}
// stateCallDeadline bounds each State Store call the daemon makes without a
// deadline of its own.
const stateCallDeadline = 20 * time.Second
// runLoopStallFactor is how many poll intervals a loop may go without
// beginning a pass before it is reported degraded.
const runLoopStallFactor = 3
// runLoopProbe reports a run loop that has stopped beginning passes, for
// instance blocked on a slow State Store.
type runLoopProbe struct {
stalest func() (string, time.Time)
cadence func() time.Duration
}
func (runLoopProbe) Name() string { return "run_loop" }
func (p runLoopProbe) Check(context.Context) health.Result {
loop, last := p.stalest()
if last.IsZero() {
return health.Result{Status: health.StatusOK}
}
limit := max(p.cadence()*runLoopStallFactor, time.Minute)
if age := time.Since(last); age > limit {
return health.Result{Status: health.StatusDegraded, Detail: loop + " loop has not begun a pass for " + age.Round(time.Second).String()}
}
return health.Result{Status: health.StatusOK}
}
+7 -16
View File
@@ -1,14 +1,12 @@
// sd_notify.go wires the daemon into internal/sdnotify's implementation of
// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts
// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still
// making progress.
// serving, and a WATCHDOG=1 heartbeat while the process is alive.
//
// The protocol client and the heartbeat sampler live in internal/sdnotify
// because archie-messaging, a separate process that cannot import this
// package, is the next process that must be able to announce itself.
// What stays here is what is the daemon's own: which
// marker counts as progress (daemon.StalestLoop) and how long a loop may go
// without beginning a poll pass.
// A run loop that stalls on a slow peer is reported by the run_loop readiness
// probe, never by withholding the heartbeat.
//
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as
@@ -37,7 +35,7 @@ func (b *boot) announceReady() {
sdnotify.Ready(b.log)
}
// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does
// startWatchdog arms the heartbeat. It does
// nothing unless a manager named a notify socket and asked for a watchdog
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
func (b *boot) startWatchdog(ctx context.Context) {
@@ -52,14 +50,10 @@ func (b *boot) startWatchdog(ctx context.Context) {
sampler := sdnotify.Watchdog{
Interval: interval,
Cadence: b.loopCadence,
Progress: b.d.StalestLoop,
Beat: func() { n.Send(sdnotify.WatchdogState) },
Log: b.log,
}
b.log.InfoContext(ctx, "systemd watchdog armed",
"heartbeat", interval.String(),
"loop_cadence", b.loopCadence().String())
"heartbeat", interval.String())
go func() {
if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
@@ -68,11 +62,8 @@ func (b *boot) startWatchdog(ctx context.Context) {
}()
}
// loopCadence is the longest interval at which the run loop may go without
// beginning a fresh poll pass, and therefore how stale the progress marker may
// be before the loop counts as stalled.
//
// The shared interval is re-read so live polling changes are honoured.
// loopCadence is the interval at which the run loop begins a poll pass. The
// shared interval is re-read so live polling changes are honoured.
func (b *boot) loopCadence() time.Duration {
return b.cfgHolder.Get().PollInterval.Std()
}
+4 -2
View File
@@ -1,6 +1,8 @@
package servicekit
import (
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
@@ -15,7 +17,7 @@ func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) {
// StateStoreClient dials the State Store gRPC service as the caller service.
// Calls wait for the State Store rather than fail while it is down, so boot
// order between services does not matter.
func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) {
func StateStoreClient(caller string, cfg config.Config, options ...grpc.DialOption) (*staterpc.Client, func(), error) {
target, err := cfg.Services.RequireTarget(config.ServiceNameState)
if err != nil {
return nil, nil, err
@@ -24,5 +26,5 @@ func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(
if err != nil {
return nil, nil, err
}
return staterpc.Dial(target, caller, id, staterpc.WaitForPeer)
return staterpc.Dial(target, caller, id, append([]grpc.DialOption{staterpc.WaitForPeer}, options...)...)
}
+14
View File
@@ -1,6 +1,7 @@
package staterpc
import (
"context"
"fmt"
"strings"
"time"
@@ -93,3 +94,16 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy {
// instead of failing while it is down or restarting. A call that must not
// wait carries its own deadline.
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
// WithCallDeadline bounds every unary call that carries no deadline of its
// own, so a slow State Store fails the call instead of parking its caller.
func WithCallDeadline(d time.Duration) grpc.DialOption {
return grpc.WithChainUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
if _, ok := ctx.Deadline(); !ok {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, d)
defer cancel()
}
return invoker(ctx, method, req, reply, cc, opts...)
})
}
+7 -40
View File
@@ -1,6 +1,5 @@
// Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages.
// Heartbeats stop when the caller's progress marker goes stale. Without
// NOTIFY_SOCKET nothing is sent.
// Without NOTIFY_SOCKET nothing is sent.
package sdnotify
import (
@@ -106,58 +105,26 @@ func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Dura
return watchdog / watchdogHeartbeatDivisor, true
}
// Watchdog samples the caller's progress marker and emits the watchdog
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
// not, because it is licensed by the marker rather than by the clock.
// Watchdog emits the watchdog heartbeat on every tick for as long as the
// process can run a goroutine. It proves the process is alive, not that any
// one piece of work finished: a peer that answers slowly must degrade
// readiness, not get the process killed.
type Watchdog struct {
// Interval is the sampling cadence: half of systemd's WatchdogSec.
// Interval is the beat cadence: half of systemd's WatchdogSec.
Interval time.Duration
// Cadence reports how long the loop may go between passes.
Cadence func() time.Duration
// Progress reports the stalest loop and the start of its most recent
// pass, or the zero time when no pass has begun yet.
Progress func() (string, time.Time)
// Beat emits one WATCHDOG=1.
Beat func()
Log *slog.Logger
}
// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
// only way out: this sampler never gives up on its own, because a sampler that
// stopped would leave systemd watching a process nothing reports for.
// Run beats until ctx ends and returns ctx.Err().
func (w Watchdog) Run(ctx context.Context) error {
armedAt := time.Now()
stalled := false
ticker := time.NewTicker(w.Interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
loop, last := w.Progress()
if last.IsZero() {
// No pass has begun at all. Measure staleness from arming, so a
// loop that never starts polling is caught like one that stopped.
last = armedAt
}
cadence := w.Cadence()
switch age := time.Since(last); {
case age > cadence:
if !stalled {
w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
"loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String())
stalled = true
}
default:
if stalled {
w.Log.InfoContext(ctx, "systemd watchdog: run loop resumed; heartbeats resumed",
"last_pass_age", age.String())
stalled = false
}
w.Beat()
}
}