fix(archied): the watchdog heartbeat proves liveness and a stalled run loop reports degraded readiness (#1840)
State Store calls the daemon makes without a deadline now get one.
This commit is contained in:
6 files changed
+68
-59
No files matched your search
@@ -289,7 +289,7 @@ func (b *boot) openStateStoreAdapter(ctx context.Context) error {
|
|||||||
if target == "" {
|
if target == "" {
|
||||||
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
|
return fmt.Errorf("services.state.target is required: archied/archie-gateway no longer own archie.db; the standalone archie-state-store process owns it (docs/prds/state-store-contract.md §12 step 7)")
|
||||||
}
|
}
|
||||||
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg)
|
client, cleanup, err := servicekit.StateStoreClient(presence.Daemon, b.cfg, staterpc.WithCallDeadline(stateCallDeadline))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.log.ErrorContext(ctx, "state store adapter", "err", err)
|
b.log.ErrorContext(ctx, "state store adapter", "err", err)
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -34,6 +34,12 @@ func (b *boot) setupReadinessProbes() {
|
|||||||
_, err := b.stateStore.StatusCounts(ctx)
|
_, err := b.stateStore.StatusCounts(ctx)
|
||||||
return err
|
return err
|
||||||
}),
|
}),
|
||||||
|
runLoopProbe{stalest: func() (string, time.Time) {
|
||||||
|
if b.d == nil {
|
||||||
|
return "", time.Time{}
|
||||||
|
}
|
||||||
|
return b.d.StalestLoop()
|
||||||
|
}, cadence: b.loopCadence},
|
||||||
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
||||||
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
||||||
// The daemon no longer runs a chat channel, so it cannot observe one's
|
// The daemon no longer runs a chat channel, so it cannot observe one's
|
||||||
@@ -100,3 +106,32 @@ func (unscopedBroker) Name() string { return "nats_credential_scope" }
|
|||||||
func (unscopedBroker) Check(context.Context) health.Result {
|
func (unscopedBroker) Check(context.Context) health.Result {
|
||||||
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
|
return health.Result{Status: health.StatusDegraded, Detail: "external NATS: containers get a broker token that reaches every subject; use the embedded broker"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stateCallDeadline bounds each State Store call the daemon makes without a
|
||||||
|
// deadline of its own.
|
||||||
|
const stateCallDeadline = 20 * time.Second
|
||||||
|
|
||||||
|
// runLoopStallFactor is how many poll intervals a loop may go without
|
||||||
|
// beginning a pass before it is reported degraded.
|
||||||
|
const runLoopStallFactor = 3
|
||||||
|
|
||||||
|
// runLoopProbe reports a run loop that has stopped beginning passes, for
|
||||||
|
// instance blocked on a slow State Store.
|
||||||
|
type runLoopProbe struct {
|
||||||
|
stalest func() (string, time.Time)
|
||||||
|
cadence func() time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runLoopProbe) Name() string { return "run_loop" }
|
||||||
|
|
||||||
|
func (p runLoopProbe) Check(context.Context) health.Result {
|
||||||
|
loop, last := p.stalest()
|
||||||
|
if last.IsZero() {
|
||||||
|
return health.Result{Status: health.StatusOK}
|
||||||
|
}
|
||||||
|
limit := max(p.cadence()*runLoopStallFactor, time.Minute)
|
||||||
|
if age := time.Since(last); age > limit {
|
||||||
|
return health.Result{Status: health.StatusDegraded, Detail: loop + " loop has not begun a pass for " + age.Round(time.Second).String()}
|
||||||
|
}
|
||||||
|
return health.Result{Status: health.StatusOK}
|
||||||
|
}
|
||||||
@@ -1,14 +1,12 @@
|
|||||||
// sd_notify.go wires the daemon into internal/sdnotify's implementation of
|
// sd_notify.go wires the daemon into internal/sdnotify's implementation of
|
||||||
// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts
|
// systemd's sd_notify(3) protocol: one READY=1 when the run loop starts
|
||||||
// serving, and a WATCHDOG=1 heartbeat while the run loop is demonstrably still
|
// serving, and a WATCHDOG=1 heartbeat while the process is alive.
|
||||||
// making progress.
|
|
||||||
//
|
//
|
||||||
// The protocol client and the heartbeat sampler live in internal/sdnotify
|
// The protocol client and the heartbeat sampler live in internal/sdnotify
|
||||||
// because archie-messaging, a separate process that cannot import this
|
// because archie-messaging, a separate process that cannot import this
|
||||||
// package, is the next process that must be able to announce itself.
|
// package, is the next process that must be able to announce itself.
|
||||||
// What stays here is what is the daemon's own: which
|
// A run loop that stalls on a slow peer is reported by the run_loop readiness
|
||||||
// marker counts as progress (daemon.StalestLoop) and how long a loop may go
|
// probe, never by withholding the heartbeat.
|
||||||
// without beginning a poll pass.
|
|
||||||
//
|
//
|
||||||
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
|
// Nothing here is fatal -- a process with no NOTIFY_SOCKET (a plain terminal,
|
||||||
// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as
|
// Docker, a user unit without WatchdogSec) sends nothing and serves exactly as
|
||||||
@@ -37,7 +35,7 @@ func (b *boot) announceReady() {
|
|||||||
sdnotify.Ready(b.log)
|
sdnotify.Ready(b.log)
|
||||||
}
|
}
|
||||||
|
|
||||||
// startWatchdog arms the heartbeat sampler for the daemon's run loop. It does
|
// startWatchdog arms the heartbeat. It does
|
||||||
// nothing unless a manager named a notify socket and asked for a watchdog
|
// nothing unless a manager named a notify socket and asked for a watchdog
|
||||||
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
|
// (WATCHDOG_USEC), so every deployment without systemd is unaffected.
|
||||||
func (b *boot) startWatchdog(ctx context.Context) {
|
func (b *boot) startWatchdog(ctx context.Context) {
|
||||||
@@ -52,14 +50,10 @@ func (b *boot) startWatchdog(ctx context.Context) {
|
|||||||
|
|
||||||
sampler := sdnotify.Watchdog{
|
sampler := sdnotify.Watchdog{
|
||||||
Interval: interval,
|
Interval: interval,
|
||||||
Cadence: b.loopCadence,
|
|
||||||
Progress: b.d.StalestLoop,
|
|
||||||
Beat: func() { n.Send(sdnotify.WatchdogState) },
|
Beat: func() { n.Send(sdnotify.WatchdogState) },
|
||||||
Log: b.log,
|
|
||||||
}
|
}
|
||||||
b.log.InfoContext(ctx, "systemd watchdog armed",
|
b.log.InfoContext(ctx, "systemd watchdog armed",
|
||||||
"heartbeat", interval.String(),
|
"heartbeat", interval.String())
|
||||||
"loop_cadence", b.loopCadence().String())
|
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
if err := sampler.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||||
@@ -68,11 +62,8 @@ func (b *boot) startWatchdog(ctx context.Context) {
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
// loopCadence is the longest interval at which the run loop may go without
|
// loopCadence is the interval at which the run loop begins a poll pass. The
|
||||||
// beginning a fresh poll pass, and therefore how stale the progress marker may
|
// shared interval is re-read so live polling changes are honoured.
|
||||||
// be before the loop counts as stalled.
|
|
||||||
//
|
|
||||||
// The shared interval is re-read so live polling changes are honoured.
|
|
||||||
func (b *boot) loopCadence() time.Duration {
|
func (b *boot) loopCadence() time.Duration {
|
||||||
return b.cfgHolder.Get().PollInterval.Std()
|
return b.cfgHolder.Get().PollInterval.Std()
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
package servicekit
|
package servicekit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"google.golang.org/grpc"
|
||||||
|
|
||||||
"github.com/samcharles93/archie-core/internal/config"
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||||
@@ -15,7 +17,7 @@ func Identity(cfg config.TLS, service string) (*rpctls.Identity, error) {
|
|||||||
// StateStoreClient dials the State Store gRPC service as the caller service.
|
// StateStoreClient dials the State Store gRPC service as the caller service.
|
||||||
// Calls wait for the State Store rather than fail while it is down, so boot
|
// Calls wait for the State Store rather than fail while it is down, so boot
|
||||||
// order between services does not matter.
|
// order between services does not matter.
|
||||||
func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(), error) {
|
func StateStoreClient(caller string, cfg config.Config, options ...grpc.DialOption) (*staterpc.Client, func(), error) {
|
||||||
target, err := cfg.Services.RequireTarget(config.ServiceNameState)
|
target, err := cfg.Services.RequireTarget(config.ServiceNameState)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@@ -24,5 +26,5 @@ func StateStoreClient(caller string, cfg config.Config) (*staterpc.Client, func(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
return staterpc.Dial(target, caller, id, staterpc.WaitForPeer)
|
return staterpc.Dial(target, caller, id, append([]grpc.DialOption{staterpc.WaitForPeer}, options...)...)
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package staterpc
|
package staterpc
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -93,3 +94,16 @@ func ServerKeepalivePolicy() keepalive.EnforcementPolicy {
|
|||||||
// instead of failing while it is down or restarting. A call that must not
|
// instead of failing while it is down or restarting. A call that must not
|
||||||
// wait carries its own deadline.
|
// wait carries its own deadline.
|
||||||
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
|
var WaitForPeer = grpc.WithDefaultCallOptions(grpc.WaitForReady(true))
|
||||||
|
|
||||||
|
// WithCallDeadline bounds every unary call that carries no deadline of its
|
||||||
|
// own, so a slow State Store fails the call instead of parking its caller.
|
||||||
|
func WithCallDeadline(d time.Duration) grpc.DialOption {
|
||||||
|
return grpc.WithChainUnaryInterceptor(func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
|
||||||
|
if _, ok := ctx.Deadline(); !ok {
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
ctx, cancel = context.WithTimeout(ctx, d)
|
||||||
|
defer cancel()
|
||||||
|
}
|
||||||
|
return invoker(ctx, method, req, reply, cc, opts...)
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
// Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages.
|
// Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages.
|
||||||
// Heartbeats stop when the caller's progress marker goes stale. Without
|
// Without NOTIFY_SOCKET nothing is sent.
|
||||||
// NOTIFY_SOCKET nothing is sent.
|
|
||||||
package sdnotify
|
package sdnotify
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -106,58 +105,26 @@ func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Dura
|
|||||||
return watchdog / watchdogHeartbeatDivisor, true
|
return watchdog / watchdogHeartbeatDivisor, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// Watchdog samples the caller's progress marker and emits the watchdog
|
// Watchdog emits the watchdog heartbeat on every tick for as long as the
|
||||||
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
|
// process can run a goroutine. It proves the process is alive, not that any
|
||||||
// not, because it is licensed by the marker rather than by the clock.
|
// one piece of work finished: a peer that answers slowly must degrade
|
||||||
|
// readiness, not get the process killed.
|
||||||
type Watchdog struct {
|
type Watchdog struct {
|
||||||
// Interval is the sampling cadence: half of systemd's WatchdogSec.
|
// Interval is the beat cadence: half of systemd's WatchdogSec.
|
||||||
Interval time.Duration
|
Interval time.Duration
|
||||||
// Cadence reports how long the loop may go between passes.
|
|
||||||
Cadence func() time.Duration
|
|
||||||
// Progress reports the stalest loop and the start of its most recent
|
|
||||||
// pass, or the zero time when no pass has begun yet.
|
|
||||||
Progress func() (string, time.Time)
|
|
||||||
// Beat emits one WATCHDOG=1.
|
// Beat emits one WATCHDOG=1.
|
||||||
Beat func()
|
Beat func()
|
||||||
Log *slog.Logger
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
|
// Run beats until ctx ends and returns ctx.Err().
|
||||||
// only way out: this sampler never gives up on its own, because a sampler that
|
|
||||||
// stopped would leave systemd watching a process nothing reports for.
|
|
||||||
func (w Watchdog) Run(ctx context.Context) error {
|
func (w Watchdog) Run(ctx context.Context) error {
|
||||||
armedAt := time.Now()
|
|
||||||
stalled := false
|
|
||||||
ticker := time.NewTicker(w.Interval)
|
ticker := time.NewTicker(w.Interval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
}
|
|
||||||
|
|
||||||
loop, last := w.Progress()
|
|
||||||
if last.IsZero() {
|
|
||||||
// No pass has begun at all. Measure staleness from arming, so a
|
|
||||||
// loop that never starts polling is caught like one that stopped.
|
|
||||||
last = armedAt
|
|
||||||
}
|
|
||||||
cadence := w.Cadence()
|
|
||||||
switch age := time.Since(last); {
|
|
||||||
case age > cadence:
|
|
||||||
if !stalled {
|
|
||||||
w.Log.WarnContext(ctx, "systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
|
|
||||||
"loop", loop, "stale_for", age.String(), "loop_cadence", cadence.String())
|
|
||||||
stalled = true
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
if stalled {
|
|
||||||
w.Log.InfoContext(ctx, "systemd watchdog: run loop resumed; heartbeats resumed",
|
|
||||||
"last_pass_age", age.String())
|
|
||||||
stalled = false
|
|
||||||
}
|
|
||||||
w.Beat()
|
w.Beat()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user