fix(archied): the watchdog heartbeat proves liveness and a stalled run loop reports degraded readiness (#1840)
State Store calls the daemon makes without a deadline now get one.
This commit is contained in:
6 files changed
+68
-59
No files matched your search
@@ -289,7 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
||||
if target == "" {
|
||||
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
|
||||
}
|
||||
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg)
|
||||
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline))
|
||||
if err != nil {
|
||||
b.log.ErrorContext(ctx, "state store adapter", "err", err)
|
||||
return err
|
||||
|
||||
@@ -34,6 +34,12 @@ func (b *boot) setupReadinessProbes() {
|
||||
_, err := b.stateStore.StatusCounts(ctx)
|
||||
return err
|
||||
}),
|
||||
runLoopProbe{stalest: func() (string, time.Time) {
|
||||
if b.d == nil {
|
||||
return "", time.Time{}
|
||||
}
|
||||
return b.d.StalestLoop()
|
||||
}, cadence: b.loopCadence},
|
||||
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
||||
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
||||
// The daemon no longer runs a chat channel, so it cannot observe one's
|
||||
@@ -100,3 +106,32 @@ func (unscopedBroker) Name() string { return "nats_credential_scope" }
|
||||
func (unscopedBroker) Check(context.Context) health.Result {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
|
||||
}
|
||||
|
||||
// stateCallDeadline bounds each State Store call the daemon makes without a
|
||||
// deadline of its own.
|
||||
const stateCallDeadline = 20 * time.Second
|
||||
|
||||
// runLoopStallFactor is how many poll intervals a loop may go without
|
||||
// beginning a pass before it is reported degraded.
|
||||
const runLoopStallFactor = 3
|
||||
|
||||
// runLoopProbe reports a run loop that has stopped beginning passes, for
|
||||
// instance blocked on a slow State Store.
|
||||
type runLoopProbe struct {
|
||||
stalest func() (string, time.Time)
|
||||
cadence func() time.Duration
|
||||
}
|
||||
|
||||
func (runLoopProbe) Name() string { return "run_loop" }
|
||||
|
||||
func (p runLoopProbe) Check(context.Context) health.Result {
|
||||
loop, last := p.stalest()
|
||||
if last.IsZero() {
|
||||
return health.Result{Status: health.StatusOK}
|
||||
}
|
||||
limit := max(p.cadence()*runLoopStallFactor, time.Minute)
|
||||
if age := time.Since(last); age > limit {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: loop + " loop has not begun a pass for " + age.Round(time.Second).String()}
|
||||
}
|
||||
return health.Result{Status: health.StatusOK}
|
||||
}
|
||||
@@ -1,14 +1,12 @@
|
||||
// sd_notify.go wires the daemon into internal/sdnotify's implementation of
|
||||
// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts
|
||||
// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still
|
||||
// making progress.
|
||||
// serving, and a WATCHDOG=1 heartbeat while the process is alive.
|
||||
//
|
||||
// The protocol client and the heartbeat sampler live in internal/sdnotify
|
||||
// because archie-messaging, a separate process that cannot import this
|
||||
// package, is the next process that must be able to announce itself.
|
||||
// What stays here is what is the daemon's own: which
|
||||
// marker counts as progress (daemon.StalestLoop) and how long a loop may go
|
||||
// without beginning a poll pass.
|
||||
// A run loop that stalls on a slow peer is reported by the run_loop readiness
|
||||
// probe, never by withholding the heartbeat.
|
||||
//
|
||||
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
|
||||
// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as
|
||||
@@ -37,7 +35,7 @@ func (b *boot) announceReady() {
|
||||
sdnotify.Ready(b.log)
|
||||
}
|
||||
|
||||
// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does
|
||||
// startWatchdog arms the heartbeat. It does
|
||||
// nothing unless a manager named a notify socket and asked for a watchdog
|
||||
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
|
||||
func (b *boot) startWatchdog(ctx context.Context) {
|
||||
@@ -52,14 +50,10 @@ func (b *boot) startWatchdog(ctx context.Context) {
|
||||
|
||||
sampler := sdnotify.Watchdog{
|
||||
Interval: interval,
|
||||
Cadence: b.loopCadence,
|
||||
Progress: b.d.StalestLoop,
|
||||
Beat: func() { n.Send(sdnotify.WatchdogState) },
|
||||
Log: b.log,
|
||||
}
|
||||
b.log.InfoContext(ctx, "systemd watchdog armed",
|
||||
"heartbeat", interval.String(),
|
||||
"loop_cadence", b.loopCadence().String())
|
||||
"heartbeat", interval.String())
|
||||
|
||||
go func() {
|
||||
if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
@@ -68,11 +62,8 @@ func (b *boot) startWatchdog(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// loopCadence is the longest interval at which the run loop may go without
|
||||
// beginning a fresh poll pass, and therefore how stale the progress marker may
|
||||
// be before the loop counts as stalled.
|
||||
//
|
||||
// The shared interval is re-read so live polling changes are honoured.
|
||||
// loopCadence is the interval at which the run loop begins a poll pass. The
|
||||
// shared interval is re-read so live polling changes are honoured.
|
||||
func (b *boot) loopCadence() time.Duration {
|
||||
return b.cfgHolder.Get().PollInterval.Std()
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package servicekit
|
||||
|
||||
import (
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
@@ -15,7 +17,7 @@ func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) {
|
||||
// StateStoreClient dials the State Store gRPC service as the caller service.
|
||||
// Calls wait for the State Store rather than fail while it is down, so boot
|
||||
// order between services does not matter.
|
||||
func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) {
|
||||
func StateStoreClient(caller string, cfg config.Config, options ...grpc.DialOption) (*staterpc.Client, func(), error) {
|
||||
target, err := cfg.Services.RequireTarget(config.ServiceNameState)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -24,5 +26,5 @@ func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return staterpc.Dial(target, caller, id, staterpc.WaitForPeer)
|
||||
return staterpc.Dial(target, caller, id, append([]grpc.DialOption{staterpc.WaitForPeer}, options...)...)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package staterpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -93,3 +94,16 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy {
|
||||
// instead of failing while it is down or restarting. A call that must not
|
||||
// wait carries its own deadline.
|
||||
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
|
||||
|
||||
// WithCallDeadline bounds every unary call that carries no deadline of its
|
||||
// own, so a slow State Store fails the call instead of parking its caller.
|
||||
func WithCallDeadline(d time.Duration) grpc.DialOption {
|
||||
return grpc.WithChainUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, d)
|
||||
defer cancel()
|
||||
}
|
||||
return invoker(ctx, method, req, reply, cc, opts...)
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
// Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages.
|
||||
// Heartbeats stop when the caller's progress marker goes stale. Without
|
||||
// NOTIFY_SOCKET nothing is sent.
|
||||
// Without NOTIFY_SOCKET nothing is sent.
|
||||
package sdnotify
|
||||
|
||||
import (
|
||||
@@ -106,58 +105,26 @@ func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Dura
|
||||
return watchdog / watchdogHeartbeatDivisor, true
|
||||
}
|
||||
|
||||
// Watchdog samples the caller's progress marker and emits the watchdog
|
||||
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
|
||||
// not, because it is licensed by the marker rather than by the clock.
|
||||
// Watchdog emits the watchdog heartbeat on every tick for as long as the
|
||||
// process can run a goroutine. It proves the process is alive, not that any
|
||||
// one piece of work finished: a peer that answers slowly must degrade
|
||||
// readiness, not get the process killed.
|
||||
type Watchdog struct {
|
||||
// Interval is the sampling cadence: half of systemd's WatchdogSec.
|
||||
// Interval is the beat cadence: half of systemd's WatchdogSec.
|
||||
Interval time.Duration
|
||||
// Cadence reports how long the loop may go between passes.
|
||||
Cadence func() time.Duration
|
||||
// Progress reports the stalest loop and the start of its most recent
|
||||
// pass, or the zero time when no pass has begun yet.
|
||||
Progress func() (string, time.Time)
|
||||
// Beat emits one WATCHDOG=1.
|
||||
Beat func()
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
|
||||
// only way out: this sampler never gives up on its own, because a sampler that
|
||||
// stopped would leave systemd watching a process nothing reports for.
|
||||
// Run beats until ctx ends and returns ctx.Err().
|
||||
func (w Watchdog) Run(ctx context.Context) error {
|
||||
armedAt := time.Now()
|
||||
stalled := false
|
||||
ticker := time.NewTicker(w.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
|
||||
loop, last := w.Progress()
|
||||
if last.IsZero() {
|
||||
// No pass has begun at all. Measure staleness from arming, so a
|
||||
// loop that never starts polling is caught like one that stopped.
|
||||
last = armedAt
|
||||
}
|
||||
cadence := w.Cadence()
|
||||
switch age := time.Since(last); {
|
||||
case age > cadence:
|
||||
if !stalled {
|
||||
w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
|
||||
"loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String())
|
||||
stalled = true
|
||||
}
|
||||
default:
|
||||
if stalled {
|
||||
w.Log.InfoContext(ctx, "systemd watchdog: run loop resumed; heartbeats resumed",
|
||||
"last_pass_age", age.String())
|
||||
stalled = false
|
||||
}
|
||||
w.Beat()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user