165 lines
5.3 KiB
Go
165 lines
5.3 KiB
Go
// Package sdnotify sends systemd sd_notify READY=1 and WATCHDOG=1 messages.
|
|
// Heartbeats stop when the caller's progress marker goes stale. Without
|
|
// NOTIFY_SOCKET nothing is sent.
|
|
package sdnotify
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
// NotifySocketEnv names the socket a manager passes to a Type=notify unit.
|
|
NotifySocketEnv = "NOTIFY_SOCKET"
|
|
// WatchdogUsecEnv carries WatchdogSec in microseconds.
|
|
WatchdogUsecEnv = "WATCHDOG_USEC"
|
|
// WatchdogPIDEnv names the process whose beats the watchdog counts.
|
|
WatchdogPIDEnv = "WATCHDOG_PID"
|
|
|
|
// ReadyState and WatchdogState are the two sd_notify states a process
|
|
// announces: it has started serving, and it is still making progress.
|
|
ReadyState = "READY=1"
|
|
WatchdogState = "WATCHDOG=1"
|
|
|
|
// watchdogHeartbeatDivisor is sd_notify(3)'s recommendation: beat at half
|
|
// of WatchdogSec, so one missed beat is tolerated and two are not.
|
|
watchdogHeartbeatDivisor = 2
|
|
)
|
|
|
|
// Notifier writes sd_notify states to the socket named in the environment. A
|
|
// notifier with no address -- the normal case away from systemd -- sends
|
|
// nothing and says nothing.
|
|
type Notifier struct {
|
|
addr string
|
|
log *slog.Logger
|
|
}
|
|
|
|
// New resolves the notify socket. The address is read from the environment
|
|
// systemd sets for the process, so systemd replaces the previous unit's socket
|
|
// with the new one across a restart.
|
|
func New(getenv func(string) string, log *slog.Logger) Notifier {
|
|
return Notifier{addr: getenv(NotifySocketEnv), log: log}
|
|
}
|
|
|
|
// Enabled reports whether a manager named a notify socket at all. It is the
|
|
// one fact a caller needs to decide whether a heartbeat is worth arming: with
|
|
// no socket every send is a no-op, and a sampler that cannot deliver a beat
|
|
// should not exist to withhold one.
|
|
func (n Notifier) Enabled() bool { return n.addr != "" }
|
|
|
|
// Send delivers one state line. A failure is logged, never returned: a
|
|
// notification is an observation about the process, and no observation is
|
|
// worth refusing to serve over.
|
|
func (n Notifier) Send(state string) {
|
|
if n.addr == "" {
|
|
return
|
|
}
|
|
if err := sendNotify(n.addr, state); err != nil {
|
|
n.log.Warn("systemd notify failed", "state", state, "socket", n.addr, "err", err)
|
|
}
|
|
}
|
|
|
|
// sendNotify sends one datagram to the notify socket; '@' means the
|
|
// abstract namespace.
|
|
func sendNotify(addr, state string) error {
|
|
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: addr, Net: "unixgram"})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = conn.Close() }()
|
|
if _, err := conn.Write([]byte(state)); err != nil {
|
|
return fmt.Errorf("write %q: %w", state, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Ready sends READY=1.
|
|
func Ready(log *slog.Logger) {
|
|
New(os.Getenv, log).Send(ReadyState)
|
|
}
|
|
|
|
// Heartbeat returns half of WatchdogSec. ok is false when no watchdog is
|
|
// requested or WATCHDOG_PID is another process.
|
|
func Heartbeat(getenv func(string) string, pid int, log *slog.Logger) (time.Duration, bool) {
|
|
raw := getenv(WatchdogUsecEnv)
|
|
if raw == "" {
|
|
return 0, false
|
|
}
|
|
if rawPID := getenv(WatchdogPIDEnv); rawPID != "" {
|
|
wantPID, err := strconv.Atoi(rawPID)
|
|
if err != nil || wantPID != pid {
|
|
log.Warn("systemd watchdog disabled: WATCHDOG_PID is not this process",
|
|
"watchdog_pid", rawPID, "pid", pid, "err", err)
|
|
return 0, false
|
|
}
|
|
}
|
|
watchdog, err := time.ParseDuration(raw + "us")
|
|
if err != nil || watchdog <= 0 {
|
|
log.Warn("systemd watchdog disabled: WATCHDOG_USEC unreadable", "value", raw, "err", err)
|
|
return 0, false
|
|
}
|
|
return watchdog / watchdogHeartbeatDivisor, true
|
|
}
|
|
|
|
// Watchdog samples the caller's progress marker and emits the watchdog
|
|
// heartbeat only while that marker is fresh. Sampling is periodic; the beat is
|
|
// not, because it is licensed by the marker rather than by the clock.
|
|
type Watchdog struct {
|
|
// Interval is the sampling cadence: half of systemd's WatchdogSec.
|
|
Interval time.Duration
|
|
// Cadence reports how long the loop may go between passes.
|
|
Cadence func() time.Duration
|
|
// Progress reports the start of the loop's most recent pass, or the zero
|
|
// time when no pass has begun yet.
|
|
Progress func() time.Time
|
|
// Beat emits one WATCHDOG=1.
|
|
Beat func()
|
|
Log *slog.Logger
|
|
}
|
|
|
|
// Run samples until ctx ends. It returns ctx.Err() on shutdown, which is the
|
|
// only way out: this sampler never gives up on its own, because a sampler that
|
|
// stopped would leave systemd watching a process nothing reports for.
|
|
func (w Watchdog) Run(ctx context.Context) error {
|
|
armedAt := time.Now()
|
|
stalled := false
|
|
ticker := time.NewTicker(w.Interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-ticker.C:
|
|
}
|
|
|
|
last := w.Progress()
|
|
if last.IsZero() {
|
|
// No pass has begun at all. Measure staleness from arming, so a
|
|
// loop that never starts polling is caught like one that stopped.
|
|
last = armedAt
|
|
}
|
|
cadence := w.Cadence()
|
|
switch age := time.Since(last); {
|
|
case age > cadence:
|
|
if !stalled {
|
|
w.Log.Warn("systemd watchdog: run loop has not begun a poll pass within its interval; withholding the heartbeat until it does",
|
|
"stale_for", age.String(), "loop_cadence", cadence.String())
|
|
stalled = true
|
|
}
|
|
default:
|
|
if stalled {
|
|
w.Log.Info("systemd watchdog: run loop resumed; heartbeats resumed",
|
|
"last_pass_age", age.String())
|
|
stalled = false
|
|
}
|
|
w.Beat()
|
|
}
|
|
}
|
|
}
|