feat(drain): honour external drain marker with instantiation epoch (#718)
Implement drain control via an external marker file. The daemon polls
$XDG_CONFIG_HOME/archie/.drain_request.json (the repo's config-home
convention) for a drain request, validates it against the current
instantiation epoch (kernel boot id + PID 1 start time) and, only when
the marker is live, cancels the boot context to trigger the same
graceful shutdown an operator SIGTERM does.
A marker written against a prior instantiation (reboot or PID 1/container
restart) is stale and ignored, so a leftover request never drains a
daemon that came up after it was written. An unreadable marker or epoch
fails closed to no drain. The daemon never invents a new shutdown path.
domain/drain owns the pure decision (Epoch, Marker, Decide);
infrastructure/drain reads the marker file and /proc boot facts;
app/archied wires a poller that cancels the root context on a valid
request.
Path note: the epic named ~/.archie/.drain_request.json, but archied
resolves every per-user path under ${XDG_CONFIG_HOME:-~/.config}/archie
(see AGENTS.md 'Config path'), so the marker lives there too rather than
at a path this daemon never uses.
Refs bead archie-core-1786637496817-266-06b64a72 (drain control: marker
file and instantiation epoch), part of the operations epic
archie-core-1786637496703-262-1e49aef1.
Tests: table-driven Decide (epoch mismatch -> stale, matching -> valid,
nil -> none, unreadable -> stale), marker round-trip from operator JSON,
and monitor (valid -> shutdown, stale/none/error -> no shutdown).
This commit is contained in:
1 parent
2dd095c8a7
commit
e39467b83e
8 files changed
+808
-1
No files matched your search
@@ -151,6 +151,12 @@ type boot struct {
|
||||
// Observe/Snapshot calls once tasks start completing.
|
||||
agentStatus *daemon.AgentStatus
|
||||
|
||||
// shutdown cancels the daemon's root context, the exact path a SIGTERM
|
||||
// takes to graceful shutdown. The drain monitor invokes it when it honours
|
||||
// an external drain request, so a marker-driven drain reuses the operator
|
||||
// signal path rather than inventing a new one.
|
||||
shutdown context.CancelFunc
|
||||
|
||||
cleanups []func()
|
||||
}
|
||||
|
||||
@@ -1629,6 +1635,10 @@ func (b *boot) runLoop(ctx context.Context, once bool) error {
|
||||
return nil
|
||||
}
|
||||
b.log.Info("archied running", "repos", len(b.cfg.Repos), "poll", b.cfg.PollInterval.Std().String(), "label", b.cfg.Label)
|
||||
// Drain monitoring only matters while the daemon stays up to be drained;
|
||||
// a --once cycle exits immediately, so a drain that arrives mid-cycle is
|
||||
// out of scope and the monitor is not started.
|
||||
b.startDrainMonitor(ctx)
|
||||
if err := b.d.Run(ctx); err != nil && ctx.Err() == nil {
|
||||
b.log.Error("daemon exited", "err", err)
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/drain"
|
||||
drainio "github.com/samcharles93/archie-core/internal/infrastructure/drain"
|
||||
)
|
||||
|
||||
// defaultDrainPollInterval is how often the daemon looks for a new drain
|
||||
// request. It is deliberately short (a few seconds) so an operator-triggered
|
||||
// drain is honoured promptly, while the cost is one cheap file read per tick.
|
||||
const defaultDrainPollInterval = 5 * time.Second
|
||||
|
||||
// DefaultDrainRequestPath returns the marker file path the daemon watches. It
|
||||
// lives under the repo's config home (${XDG_CONFIG_HOME:-~/.config}/archie) --
|
||||
// the established convention for archied's per-user paths -- rather than the
|
||||
// epic's literal ~/.archie, which is not a path this daemon ever uses. The
|
||||
// discrepancy is intentional: every other archied path reads from the XDG
|
||||
// config home, so the drain marker must too, or an operator pointing at the
|
||||
// documented location would miss it.
|
||||
func DefaultDrainRequestPath() string {
|
||||
return filepath.Join(configHome(), "archie", drainio.DefaultMarkerFilename)
|
||||
}
|
||||
|
||||
// monitorDrainRequests polls check for a live drain request and invokes
|
||||
// shutdown when one is honoured. It returns nil once shutdown has been
|
||||
// triggered (or when the context is cancelled), and never returns a poll error:
|
||||
// a transient read failure is logged and ignored so a broken marker or a
|
||||
// momentary read error cannot halt the monitor and silently unmount drain
|
||||
// detection.
|
||||
//
|
||||
// check reports the drain decision; shutdown is the daemon's graceful-stop
|
||||
// trigger. Both are injected so tests can exercise the monitor without touching
|
||||
// the real filesystem or signalling the process.
|
||||
func monitorDrainRequests(
|
||||
ctx context.Context,
|
||||
check func() (drain.Decision, error),
|
||||
shutdown func(),
|
||||
log *slog.Logger,
|
||||
interval time.Duration,
|
||||
) error {
|
||||
for {
|
||||
decision, err := check()
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Warn("drain request check failed", "err", err)
|
||||
case decision == drain.DecisionValid:
|
||||
log.Info("drain request accepted; initiating graceful shutdown")
|
||||
shutdown()
|
||||
return nil
|
||||
case decision == drain.DecisionStale:
|
||||
log.Warn("stale drain request ignored; marker predates current instantiation")
|
||||
}
|
||||
|
||||
timer := time.NewTimer(interval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startDrainMonitor launches the drain-monitoring goroutine for the long-lived
|
||||
// daemon. The marker path is resolved from the repo's config home; the reader
|
||||
// uses the real current instantiation epoch; and a valid request cancels the
|
||||
// boot context, which is the exact path a SIGTERM takes to graceful shutdown.
|
||||
//
|
||||
// It is only called from the non-once run loop, so the monitor never runs for
|
||||
// a single-cycle invocation.
|
||||
func (b *boot) startDrainMonitor(ctx context.Context) {
|
||||
if b.shutdown == nil {
|
||||
b.log.Error("drain monitor not started: no shutdown trigger wired")
|
||||
return
|
||||
}
|
||||
reader := drainio.New(DefaultDrainRequestPath(), nil)
|
||||
go func() {
|
||||
err := monitorDrainRequests(ctx,
|
||||
func() (drain.Decision, error) { return reader.Check() },
|
||||
func() { b.shutdown() },
|
||||
b.log,
|
||||
defaultDrainPollInterval,
|
||||
)
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
b.log.Error("drain monitor stopped", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/drain"
|
||||
)
|
||||
|
||||
const testInterval = 5 * time.Millisecond
|
||||
|
||||
// countShutdown returns a shutdown func plus a channel signalled on each call.
|
||||
func countShutdown(t *testing.T) (func(), *chan struct{}) {
|
||||
t.Helper()
|
||||
calls := make(chan struct{}, 1)
|
||||
return func() {
|
||||
select {
|
||||
case calls <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}, &calls
|
||||
}
|
||||
|
||||
func TestMonitorTriggersShutdownOnValidDecision(t *testing.T) {
|
||||
shutdown, calls := countShutdown(t)
|
||||
err := monitorDrainRequests(
|
||||
context.Background(),
|
||||
func() (drain.Decision, error) { return drain.DecisionValid, nil },
|
||||
shutdown,
|
||||
slog.New(slog.DiscardHandler),
|
||||
testInterval,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("monitorDrainRequests() error = %v, want nil", err)
|
||||
}
|
||||
select {
|
||||
case <-*calls:
|
||||
default:
|
||||
t.Fatal("shutdown was not invoked on a valid drain decision")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDoesNotShutdownOnStaleDecision(t *testing.T) {
|
||||
shutdown, calls := countShutdown(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// First poll returns stale; the monitor waits one interval. Cancel the
|
||||
// context before the next poll, then assert shutdown was never invoked and
|
||||
// the monitor returned the cancellation.
|
||||
go func() {
|
||||
time.Sleep(testInterval * 3)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := monitorDrainRequests(
|
||||
ctx,
|
||||
func() (drain.Decision, error) { return drain.DecisionStale, nil },
|
||||
shutdown,
|
||||
slog.New(slog.DiscardHandler),
|
||||
testInterval,
|
||||
)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("monitorDrainRequests() error = %v, want context.Canceled", err)
|
||||
}
|
||||
select {
|
||||
case <-*calls:
|
||||
t.Fatal("shutdown was invoked on a stale drain decision")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDoesNotShutdownOnNoMarker(t *testing.T) {
|
||||
shutdown, calls := countShutdown(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() {
|
||||
time.Sleep(testInterval * 3)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := monitorDrainRequests(
|
||||
ctx,
|
||||
func() (drain.Decision, error) { return drain.DecisionNone, nil },
|
||||
shutdown,
|
||||
slog.New(slog.DiscardHandler),
|
||||
testInterval,
|
||||
)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("monitorDrainRequests() error = %v, want context.Canceled", err)
|
||||
}
|
||||
select {
|
||||
case <-*calls:
|
||||
t.Fatal("shutdown was invoked with no drain marker")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorDoesNotShutdownOnCheckError(t *testing.T) {
|
||||
shutdown, calls := countShutdown(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
go func() {
|
||||
time.Sleep(testInterval * 3)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
boom := errors.New("marker unreadable")
|
||||
err := monitorDrainRequests(
|
||||
ctx,
|
||||
func() (drain.Decision, error) { return drain.DecisionStale, boom },
|
||||
shutdown,
|
||||
slog.New(slog.DiscardHandler),
|
||||
testInterval,
|
||||
)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("monitorDrainRequests() error = %v, want context.Canceled", err)
|
||||
}
|
||||
select {
|
||||
case <-*calls:
|
||||
t.Fatal("shutdown was invoked despite a check error")
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitorShutdownAfterStaleThenValid(t *testing.T) {
|
||||
shutdown, calls := countShutdown(t)
|
||||
callsMade := 0
|
||||
err := monitorDrainRequests(
|
||||
context.Background(),
|
||||
func() (drain.Decision, error) {
|
||||
callsMade++
|
||||
if callsMade == 1 {
|
||||
return drain.DecisionStale, nil
|
||||
}
|
||||
return drain.DecisionValid, nil
|
||||
},
|
||||
shutdown,
|
||||
slog.New(slog.DiscardHandler),
|
||||
testInterval,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("monitorDrainRequests() error = %v, want nil", err)
|
||||
}
|
||||
select {
|
||||
case <-*calls:
|
||||
default:
|
||||
t.Fatal("shutdown was not invoked after a marker became valid")
|
||||
}
|
||||
}
|
||||
@@ -350,7 +350,8 @@ func Run() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
root, cancelRoot := context.WithCancel(context.Background())
|
||||
ctx, stop := signal.NotifyContext(root, os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
b := newBootstrap()
|
||||
@@ -359,6 +360,9 @@ func Run() int {
|
||||
// as a backstop if graceful shutdown hangs past the drain-plus-grace
|
||||
// leash, and stands down once every subsystem has shut down cleanly.
|
||||
b.startShutdownWatchdog(ctx)
|
||||
// The drain monitor cancels this root to trigger a graceful shutdown, which
|
||||
// propagates through ctx exactly as an operator SIGTERM does.
|
||||
b.shutdown = cancelRoot
|
||||
if err := b.loadConfig(ctx, args.cfgPath, args.overlayPath, args.noConfigOverlay); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
// Package drain owns the decision logic for external drain requests.
|
||||
//
|
||||
// A drain is an operator-initiated request for the daemon to stop accepting
|
||||
// new work and shut itself down gracefully. It arrives out-of-band through a
|
||||
// marker file (an external trigger such as a maintenance window or a rolling
|
||||
// deploy writes it) rather than a signal, because a marker survives the phase
|
||||
// in which the process is between restarts and can be validated against the
|
||||
// *current* instantiation.
|
||||
//
|
||||
// The single source of confusion this package rules out is the stale marker: a
|
||||
// drain request written before the daemon last restarted must never drain the
|
||||
// daemon that came up afterwards. That is handled by pairing each marker with
|
||||
// an instantiation epoch (the OS boot id plus PID 1's start time) and only
|
||||
// honouring a marker whose epoch matches the current one.
|
||||
package drain
|
||||
|
||||
// Epoch identifies one instantiation of the daemon's process tree. It
|
||||
// combines the OS boot id with PID 1's start time (in clock ticks since
|
||||
// boot). Two Epochs are equal only when both parts match, so a marker written
|
||||
// before a reboot or before a PID 1 restart within the same boot is never
|
||||
// mistaken for one written against the current instantiation.
|
||||
type Epoch struct {
|
||||
// BootID is the content of /proc/sys/kernel/random/boot_id, a UUID that
|
||||
// changes on every boot.
|
||||
BootID string `json:"boot_id"`
|
||||
// PID1Start is field 22 (starttime) of /proc/1/stat, in clock ticks since
|
||||
// boot. It distinguishes a container or PID 1 restart that happened
|
||||
// without a reboot.
|
||||
PID1Start int64 `json:"pid1_start"`
|
||||
}
|
||||
|
||||
// Equal reports whether e and other identify the same instantiation.
|
||||
func (e Epoch) Equal(other Epoch) bool {
|
||||
return e.BootID == other.BootID && e.PID1Start == other.PID1Start
|
||||
}
|
||||
|
||||
// Empty reports whether the epoch carries no usable identity facts. An empty
|
||||
// epoch cannot validate any marker, so callers must fail closed (never drain)
|
||||
// rather than trust the marker.
|
||||
func (e Epoch) Empty() bool {
|
||||
return e.BootID == "" || e.PID1Start <= 0
|
||||
}
|
||||
|
||||
// Marker is the parsed .drain_request.json written by an external drain
|
||||
// trigger. Only InstantiationEpoch is load-bearing for the decision; the
|
||||
// optional operator fields (Reason, RequestedAt) are informational and do not
|
||||
// influence whether the request is honoured.
|
||||
type Marker struct {
|
||||
// InstantiationEpoch records the epoch of the daemon the request targets.
|
||||
InstantiationEpoch Epoch `json:"instantiation_epoch"`
|
||||
// Reason is an optional operator note (e.g. "maintenance window").
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// RequestedAt is an optional RFC 3339 timestamp.
|
||||
RequestedAt string `json:"requested_at,omitempty"`
|
||||
}
|
||||
|
||||
// Decision is the validation outcome for a marker against the current epoch.
|
||||
type Decision int
|
||||
|
||||
const (
|
||||
// DecisionNone means no drain request is present (no marker, or the source
|
||||
// reported nothing). No drain.
|
||||
DecisionNone Decision = iota
|
||||
// DecisionStale means a marker is present but its epoch does not match the
|
||||
// current instantiation -- it was written before the last restart and must
|
||||
// be ignored. No drain.
|
||||
DecisionStale
|
||||
// DecisionValid means a marker is present and its epoch matches the current
|
||||
// instantiation: a live drain request that must trigger graceful shutdown.
|
||||
DecisionValid
|
||||
)
|
||||
|
||||
func (d Decision) String() string {
|
||||
switch d {
|
||||
case DecisionNone:
|
||||
return "none"
|
||||
case DecisionStale:
|
||||
return "stale"
|
||||
case DecisionValid:
|
||||
return "valid"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// Decide returns the drain decision for marker against current.
|
||||
//
|
||||
// A nil marker means no request (DecisionNone). An empty current epoch means
|
||||
// the OS identity facts could not be established, so the request is treated as
|
||||
// stale -- a daemon that cannot prove the marker is live must not shut itself
|
||||
// down on it. Only a marker whose epoch exactly matches a non-empty current
|
||||
// epoch is honoured.
|
||||
func Decide(current Epoch, marker *Marker) Decision {
|
||||
if marker == nil {
|
||||
return DecisionNone
|
||||
}
|
||||
if current.Empty() {
|
||||
return DecisionStale
|
||||
}
|
||||
if !marker.InstantiationEpoch.Equal(current) {
|
||||
return DecisionStale
|
||||
}
|
||||
return DecisionValid
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package drain
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDecideTable(t *testing.T) {
|
||||
bootA := Epoch{BootID: "boot-a", PID1Start: 100}
|
||||
bootB := Epoch{BootID: "boot-b", PID1Start: 100}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
current Epoch
|
||||
marker *Marker
|
||||
want Decision
|
||||
}{
|
||||
{
|
||||
name: "no marker reports none",
|
||||
current: bootA,
|
||||
marker: nil,
|
||||
want: DecisionNone,
|
||||
},
|
||||
{
|
||||
name: "marker matching current epoch is valid",
|
||||
current: bootA,
|
||||
marker: &Marker{InstantiationEpoch: bootA},
|
||||
want: DecisionValid,
|
||||
},
|
||||
{
|
||||
name: "marker written against a prior boot is stale",
|
||||
current: bootA,
|
||||
marker: &Marker{InstantiationEpoch: bootB},
|
||||
want: DecisionStale,
|
||||
},
|
||||
{
|
||||
name: "marker written against a prior pid1 start within same boot is stale",
|
||||
current: bootA,
|
||||
marker: &Marker{InstantiationEpoch: Epoch{BootID: "boot-a", PID1Start: 99}},
|
||||
want: DecisionStale,
|
||||
},
|
||||
{
|
||||
name: "marker still valid when operator fields present",
|
||||
current: bootA,
|
||||
marker: &Marker{InstantiationEpoch: bootA, Reason: "maintenance", RequestedAt: "2026-09-05T00:00:00Z"},
|
||||
want: DecisionValid,
|
||||
},
|
||||
{
|
||||
name: "empty current epoch fails closed to stale even on matching marker",
|
||||
current: Epoch{},
|
||||
marker: &Marker{InstantiationEpoch: bootA},
|
||||
want: DecisionStale,
|
||||
},
|
||||
{
|
||||
name: "zero pid1 start in current epoch fails closed to stale",
|
||||
current: Epoch{BootID: "boot-a", PID1Start: 0},
|
||||
marker: &Marker{InstantiationEpoch: Epoch{BootID: "boot-a", PID1Start: 0}},
|
||||
want: DecisionStale,
|
||||
},
|
||||
{
|
||||
name: "empty boot id in current epoch fails closed to stale",
|
||||
current: Epoch{BootID: "", PID1Start: 100},
|
||||
marker: &Marker{InstantiationEpoch: Epoch{BootID: "", PID1Start: 100}},
|
||||
want: DecisionStale,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := Decide(test.current, test.marker); got != test.want {
|
||||
t.Fatalf("Decide() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpochEqual(t *testing.T) {
|
||||
if !(Epoch{BootID: "b", PID1Start: 1}).Equal(Epoch{BootID: "b", PID1Start: 1}) {
|
||||
t.Fatal("equal epochs reported unequal")
|
||||
}
|
||||
if (Epoch{BootID: "b", PID1Start: 1}).Equal(Epoch{BootID: "b", PID1Start: 2}) {
|
||||
t.Fatal("epochs differing only in pid1 start reported equal")
|
||||
}
|
||||
if (Epoch{BootID: "a", PID1Start: 1}).Equal(Epoch{BootID: "b", PID1Start: 1}) {
|
||||
t.Fatal("epochs differing only in boot id reported equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpochEmpty(t *testing.T) {
|
||||
if !(Epoch{}).Empty() {
|
||||
t.Fatal("zero epoch should be empty")
|
||||
}
|
||||
if !(Epoch{BootID: "a", PID1Start: 0}).Empty() {
|
||||
t.Fatal("zero pid1 start should be empty")
|
||||
}
|
||||
if !(Epoch{BootID: "", PID1Start: 5}).Empty() {
|
||||
t.Fatal("empty boot id should be empty")
|
||||
}
|
||||
if (Epoch{BootID: "a", PID1Start: 5}).Empty() {
|
||||
t.Fatal("well-formed epoch should not be empty")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// Package drain is the infrastructure implementation of drain-request
|
||||
// detection. It reads the external marker file from disk and the OS identity
|
||||
// facts that make up the current instantiation epoch, then hands the verdict
|
||||
// to the domain's Decide function.
|
||||
//
|
||||
// It only reads. Deciding (and any shutdown consequence) belongs to the caller,
|
||||
// so the daemon never drains itself from this package.
|
||||
package drain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/drain"
|
||||
)
|
||||
|
||||
// DefaultMarkerFilename is the marker file the daemon watches for an external
|
||||
// drain request.
|
||||
var DefaultMarkerFilename = ".drain_request.json"
|
||||
|
||||
// CurrentEpoch reads the OS identity facts that identify the current
|
||||
// instantiation: the boot id (a UUID that changes every boot) and PID 1's
|
||||
// start time (in clock ticks since boot), which distinguishes a container or
|
||||
// PID 1 restart that happened without a reboot.
|
||||
func CurrentEpoch() (drain.Epoch, error) {
|
||||
bootID, err := readBootID()
|
||||
if err != nil {
|
||||
return drain.Epoch{}, fmt.Errorf("read boot id: %w", err)
|
||||
}
|
||||
pid1Start, err := readPid1Start()
|
||||
if err != nil {
|
||||
return drain.Epoch{}, fmt.Errorf("read pid 1 start time: %w", err)
|
||||
}
|
||||
return drain.Epoch{BootID: bootID, PID1Start: pid1Start}, nil
|
||||
}
|
||||
|
||||
// readBootID returns the kernel boot id from
|
||||
// /proc/sys/kernel/random/boot_id. It returns an error when the file cannot be
|
||||
// read, so an unverifiable epoch fails closed in the caller.
|
||||
func readBootID() (string, error) {
|
||||
data, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
bootID := strings.TrimSpace(string(data))
|
||||
if bootID == "" {
|
||||
return "", errors.New("empty boot id")
|
||||
}
|
||||
return bootID, nil
|
||||
}
|
||||
|
||||
// readPid1Start returns PID 1's start time from /proc/1/stat.
|
||||
func readPid1Start() (int64, error) {
|
||||
data, err := os.ReadFile("/proc/1/stat")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return parseProcStat(string(data))
|
||||
}
|
||||
|
||||
// parseProcStat extracts PID 1's start time (field 22, starttime) from the
|
||||
// textual /proc/[pid]/stat record. The comm field (field 2) is wrapped in
|
||||
// parentheses and may itself contain spaces or parentheses, so the record is
|
||||
// split after the last closing parenthesis and field 22 is the 20th token
|
||||
// there (post-paren token 0 is field 3, the state).
|
||||
func parseProcStat(stat string) (int64, error) {
|
||||
paren := strings.LastIndexByte(stat, ')')
|
||||
if paren < 0 {
|
||||
return 0, errors.New("malformed proc stat: no closing parenthesis")
|
||||
}
|
||||
fields := strings.Fields(stat[paren+1:])
|
||||
// After the parenthesis the tokens are fields 3..N, so field 22 is
|
||||
// index 22-3 = 19.
|
||||
if len(fields) < 20 {
|
||||
return 0, fmt.Errorf("malformed proc stat: %d post-paren fields, want >= 20", len(fields))
|
||||
}
|
||||
start, err := strconv.ParseInt(fields[19], 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse pid 1 start time %q: %w", fields[19], err)
|
||||
}
|
||||
if start <= 0 {
|
||||
return 0, fmt.Errorf("invalid pid 1 start time %d", start)
|
||||
}
|
||||
return start, nil
|
||||
}
|
||||
|
||||
// Reader reads a drain marker file and the current instantiation epoch,
|
||||
// reporting whether a live drain request exists right now. The epoch source is
|
||||
// injectable so tests can exercise the full read-and-decide path without
|
||||
// touching /proc.
|
||||
type Reader struct {
|
||||
path string
|
||||
epoch func() (drain.Epoch, error)
|
||||
}
|
||||
|
||||
// New builds a Reader for the marker file at markerPath. When epoch is nil the
|
||||
// package's CurrentEpoch is used; pass a fake epoch function in tests to pin
|
||||
// the current instantiation without depending on the host.
|
||||
func New(markerPath string, epoch func() (drain.Epoch, error)) *Reader {
|
||||
if epoch == nil {
|
||||
epoch = CurrentEpoch
|
||||
}
|
||||
return &Reader{path: markerPath, epoch: epoch}
|
||||
}
|
||||
|
||||
// Check returns the drain decision for the marker at the reader's path against
|
||||
// the current epoch.
|
||||
//
|
||||
// A missing marker returns DecisionNone. The returned error is informational --
|
||||
// it exists so a caller can log why a request was not honoured -- but the
|
||||
// decision is authoritative and always fails closed on an unrecoverable read
|
||||
// (permission error, malformed JSON, unreadable epoch) so a broken marker can
|
||||
// never drain the daemon.
|
||||
func (r *Reader) Check() (drain.Decision, error) {
|
||||
data, err := os.ReadFile(r.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return drain.DecisionNone, nil
|
||||
}
|
||||
if err != nil {
|
||||
return drain.DecisionStale, fmt.Errorf("read drain marker: %w", err)
|
||||
}
|
||||
var marker drain.Marker
|
||||
if err := json.Unmarshal(data, &marker); err != nil {
|
||||
return drain.DecisionStale, fmt.Errorf("decode drain marker: %w", err)
|
||||
}
|
||||
current, err := r.epoch()
|
||||
if err != nil {
|
||||
return drain.DecisionStale, fmt.Errorf("resolve instantiation epoch: %w", err)
|
||||
}
|
||||
return drain.Decide(current, &marker), nil
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package drain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/drain"
|
||||
)
|
||||
|
||||
func TestParseProcStatField22(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stat string
|
||||
want int64
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "systemd pid 1 with real start time",
|
||||
stat: "1 (systemd) S 0 1 1 0 -1 4194560 214130 42927786 23 588 817 557 98753 15714 20 0 1 0 5 0 0 0 0 ...",
|
||||
want: 5,
|
||||
},
|
||||
{
|
||||
name: "comm containing spaces is handled",
|
||||
stat: "1 (init daemon) S 0 1 1 0 -1 4194560 0 0 0 0 0 0 0 0 20 0 1 0 42 0 0 0 0 ...",
|
||||
want: 42,
|
||||
},
|
||||
{
|
||||
name: "many trailing fields (real proc stat has 50+) are tolerated",
|
||||
stat: "1 (systemd) S 0 1 1 0 -1 4194560 214130 42927786 23 588 817 557 98753 15714 20 0 1 0 9 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0",
|
||||
want: 9,
|
||||
},
|
||||
{
|
||||
name: "too few post-paren fields fails",
|
||||
stat: "1 (systemd) S 0 1 1",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "missing closing paren fails",
|
||||
stat: "1 systemd S 0 1 1",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "non-numeric start time fails",
|
||||
stat: "1 (systemd) S 0 1 1 0 -1 4194560 214130 42927786 23 588 817 557 98753 15714 20 0 1 0 notanumber 0 0 0 0",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "zero start time fails",
|
||||
stat: "1 (systemd) S 0 1 1 0 -1 4194560 214130 42927786 23 588 817 557 98753 15714 20 0 1 0 0 0 0 0 0",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, err := parseProcStat(test.stat)
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("parseProcStat() error = %v, wantErr %v", err, test.wantErr)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("parseProcStat() = %d, want %d", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// writeMarker writes a drain marker JSON file with the given epoch.
|
||||
func writeMarker(t *testing.T, dir string, epoch drain.Epoch) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, DefaultMarkerFilename)
|
||||
payload, err := json.Marshal(drain.Marker{InstantiationEpoch: epoch, Reason: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, payload, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestReaderCheckTable(t *testing.T) {
|
||||
current := drain.Epoch{BootID: "boot-now", PID1Start: 100}
|
||||
prior := drain.Epoch{BootID: "boot-old", PID1Start: 100}
|
||||
|
||||
match := func() (drain.Epoch, error) { return current, nil }
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
write func(dir string) string
|
||||
epoch func() (drain.Epoch, error)
|
||||
want drain.Decision
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "absent marker reports none",
|
||||
write: func(dir string) string {
|
||||
return filepath.Join(dir, DefaultMarkerFilename)
|
||||
},
|
||||
epoch: match,
|
||||
want: drain.DecisionNone,
|
||||
},
|
||||
{
|
||||
name: "marker with current epoch is valid",
|
||||
write: func(dir string) string { return writeMarker(t, dir, current) },
|
||||
epoch: match,
|
||||
want: drain.DecisionValid,
|
||||
},
|
||||
{
|
||||
name: "marker written against a prior epoch is stale",
|
||||
write: func(dir string) string { return writeMarker(t, dir, prior) },
|
||||
epoch: match,
|
||||
want: drain.DecisionStale,
|
||||
},
|
||||
{
|
||||
name: "current epoch unreadable fails closed to stale",
|
||||
write: func(dir string) string { return writeMarker(t, dir, current) },
|
||||
epoch: func() (drain.Epoch, error) {
|
||||
return drain.Epoch{}, errors.New("proc unavailable")
|
||||
},
|
||||
want: drain.DecisionStale,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "malformed marker fails closed to stale",
|
||||
write: func(dir string) string {
|
||||
path := filepath.Join(dir, DefaultMarkerFilename)
|
||||
if err := os.WriteFile(path, []byte("{ not json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
},
|
||||
epoch: match,
|
||||
want: drain.DecisionStale,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "marker file is a directory fails closed to stale",
|
||||
write: func(dir string) string {
|
||||
path := filepath.Join(dir, DefaultMarkerFilename)
|
||||
if err := os.Mkdir(path, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
},
|
||||
epoch: match,
|
||||
want: drain.DecisionStale,
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := test.write(dir)
|
||||
reader := New(path, test.epoch)
|
||||
got, err := reader.Check()
|
||||
if (err != nil) != test.wantErr {
|
||||
t.Fatalf("Check() error = %v, wantErr %v", err, test.wantErr)
|
||||
}
|
||||
if got != test.want {
|
||||
t.Fatalf("Check() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderParsesOperatorWrittenJSON(t *testing.T) {
|
||||
// The marker is written by an external operator script, so the on-disk
|
||||
// shape (underscore keys) is part of the contract. Unmarshal it as-is and
|
||||
// confirm the epoch round-trips to a valid decision -- this is what guards
|
||||
// the JSON tag wiring rather than go fixture marshaling.
|
||||
current := drain.Epoch{BootID: "boot-now", PID1Start: 100}
|
||||
const raw = `{"instantiation_epoch":{"boot_id":"boot-now","pid1_start":100},"reason":"maintenance window","requested_at":"2026-09-05T00:00:00Z"}`
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, DefaultMarkerFilename)
|
||||
if err := os.WriteFile(path, []byte(raw), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := New(path, func() (drain.Epoch, error) { return current, nil }).Check()
|
||||
if err != nil {
|
||||
t.Fatalf("Check() error = %v", err)
|
||||
}
|
||||
if got != drain.DecisionValid {
|
||||
t.Fatalf("Check() = %v, want valid", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDefaultsToCurrentEpoch(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// No epoch function passed: New must fall back to the package's real
|
||||
// CurrentEpoch. The marker records the real current epoch read from /proc,
|
||||
// so Check must report it valid -- proving the default wiring works end to
|
||||
// end without a host restart.
|
||||
current, err := CurrentEpoch()
|
||||
if err != nil {
|
||||
t.Skipf("cannot read current epoch on this host: %v", err)
|
||||
}
|
||||
path := writeMarker(t, dir, current)
|
||||
got, err := New(path, nil).Check()
|
||||
if err != nil {
|
||||
t.Fatalf("Check() error = %v", err)
|
||||
}
|
||||
if got != drain.DecisionValid {
|
||||
t.Fatalf("Check() = %v, want valid", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user