Files
archie/internal/infrastructure/readiness/probes.go
T

391 lines
13 KiB
Go

// Package readiness implements the health probes.
package readiness
import (
"context"
"errors"
"fmt"
"os"
"strings"
"syscall"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/infrastructure/access"
)
// --- state_db ---
// statusCounter is the read-only store surface the state_db probe needs.
// The task store satisfies it; the probe never depends on the
// full task-lifecycle interface.
type statusCounter interface {
StatusCounts(ctx context.Context) (map[string]int, error)
}
// StoreProbe checks the state database with a cheap read-only query
// (StatusCounts). It is the store's own health signal: a lock, corruption,
// or connection failure surfaces as an error here.
type StoreProbe struct {
Store statusCounter
}
// NewStoreProbe returns a state_db probe over the given store.
func NewStoreProbe(s statusCounter) *StoreProbe {
return &StoreProbe{Store: s}
}
func (p *StoreProbe) Name() string { return "state_db" }
func (p *StoreProbe) Check(ctx context.Context) health.Result {
if p.Store == nil {
return health.Result{Status: health.StatusDegraded, Detail: "store not wired"}
}
if _, err := p.Store.StatusCounts(ctx); err != nil {
return health.Result{Status: health.StatusDegraded, Detail: "read failed: " + err.Error()}
}
return health.Result{Status: health.StatusOK}
}
// --- remote contract ---
// ContractProbe checks that a remote service answers a cheap call within a
// timeout.
type ContractProbe struct {
ProbeName string
Timeout time.Duration
Ping func(context.Context) error
}
// NewContractProbe returns a probe named name that calls ping. A zero or
// negative timeout leaves the caller's context deadline in force.
func NewContractProbe(name string, timeout time.Duration, ping func(context.Context) error) *ContractProbe {
return &ContractProbe{ProbeName: name, Timeout: timeout, Ping: ping}
}
func (p *ContractProbe) Name() string { return p.ProbeName }
func (p *ContractProbe) Check(ctx context.Context) health.Result {
if p.Ping == nil {
return health.Result{Status: health.StatusDegraded, Detail: "contract not wired"}
}
if p.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, p.Timeout)
defer cancel()
}
if err := p.Ping(ctx); err != nil {
return health.Result{Status: health.StatusDegraded, Detail: "unreachable: " + err.Error()}
}
return health.Result{Status: health.StatusOK}
}
// --- access policies ---
// PolicyProblems reports the stored access policies this process's engine
// could not validate. The policy chain is the source; a policy that failed
// startup re-validation denies its level and must be named here with its
// error.
type PolicyProblems interface {
Problems() []access.Problem
}
// ProblemProbe reports the stored policies the engine could not validate.
// It is a static probe: the engine is immutable for the life of the process,
// so the problems it names do not change under it.
type ProblemProbe struct {
ProbeName string
Source PolicyProblems
}
// NewProblemProbe returns an access_policies probe over a problem source.
// A nil source degrades the probe rather than lying about the chain.
func NewProblemProbe(name string, problems []access.Problem) *ProblemProbe {
return &ProblemProbe{ProbeName: name, Source: staticProblemSource(problems)}
}
type staticProblemSource []access.Problem
func (s staticProblemSource) Problems() []access.Problem { return s }
func (p *ProblemProbe) Name() string { return p.ProbeName }
func (p *ProblemProbe) Check(ctx context.Context) health.Result {
if p.Source == nil {
return health.Result{Status: health.StatusDegraded, Detail: "access engine not wired"}
}
problems := p.Source.Problems()
if len(problems) == 0 {
return health.Result{Status: health.StatusOK}
}
details := make([]string, 0, len(problems))
for _, problem := range problems {
details = append(details, problem.Policy.ID+": "+problem.Err.Error())
}
return health.Result{Status: health.StatusDegraded, Detail: strings.Join(details, "; ")}
}
// --- config ---
// ConfigProbe validates the running configuration. It is the same check the
// loader applies before accepting a config, run against the live snapshot.
// The validation function is injected so this probe does not import the
// configuration package directly; composition passes configuration.Validate.
type ConfigProbe struct {
Get func() config.Config
Validate func(*config.Config) error
}
// NewConfigProbe returns a config probe over the running-config getter and
// validator. Either may be nil; a nil validator degrades the probe.
func NewConfigProbe(get func() config.Config, validate func(*config.Config) error) *ConfigProbe {
return &ConfigProbe{Get: get, Validate: validate}
}
func (p *ConfigProbe) Name() string { return "config" }
func (p *ConfigProbe) Check(ctx context.Context) health.Result {
if p.Get == nil {
return health.Result{Status: health.StatusDegraded, Detail: "config not wired"}
}
cfg := p.Get()
if p.Validate == nil {
return health.Result{Status: health.StatusDegraded, Detail: "config validator not wired"}
}
if err := p.Validate(&cfg); err != nil {
return health.Result{Status: health.StatusDegraded, Detail: "invalid: " + err.Error()}
}
return health.Result{Status: health.StatusOK}
}
// --- disk ---
// DefaultDiskThreshold is the fraction of a filesystem that, when in use,
// makes the disk probe report degraded. It matches the "disk < 90% used"
// requirement of the readiness epic.
const DefaultDiskThreshold = 0.90
// DiskTarget is one filesystem location to measure. Optional targets are
// ignored when they disappear (for example, /var on a non-Linux installation).
type DiskTarget struct {
Name string
Path string
Optional bool
}
// DiskProbe checks each configured filesystem independently and aggregates the
// results. Statfs is injectable so callers can test disk boundaries without
// filling a filesystem.
type DiskProbe struct {
// Path is retained for source compatibility with the original single-path
// probe. Targets takes precedence when non-empty.
Path string
Targets []DiskTarget
Threshold float64
Statfs func(string, *syscall.Statfs_t) error
}
// NewDiskProbe returns a single-path disk probe.
func NewDiskProbe(path string) *DiskProbe {
return NewDiskProbeTargets([]DiskTarget{{Name: "work", Path: path}})
}
// NewDiskProbeTargets returns a disk probe for the supplied, stable-ordered targets.
func NewDiskProbeTargets(targets []DiskTarget) *DiskProbe {
return &DiskProbe{Targets: append([]DiskTarget(nil), targets...), Threshold: DefaultDiskThreshold, Statfs: syscall.Statfs}
}
func (p *DiskProbe) Name() string { return "disk" }
// reduceTargets keeps one target per path, in first-seen position, with the
// strictest requirement for that path.
func reduceTargets(targets []DiskTarget) []DiskTarget {
index := make(map[string]int, len(targets))
out := make([]DiskTarget, 0, len(targets))
for _, target := range targets {
if target.Path == "" {
continue
}
if at, ok := index[target.Path]; ok {
if !target.Optional && out[at].Optional {
out[at] = target
}
continue
}
index[target.Path] = len(out)
out = append(out, target)
}
return out
}
func (p *DiskProbe) Check(ctx context.Context) health.Result {
_ = ctx
targets := p.Targets
if len(targets) == 0 && p.Path != "" {
targets = []DiskTarget{{Name: "work", Path: p.Path}}
}
if len(targets) == 0 {
return health.Result{Status: health.StatusDegraded, Detail: "no disk targets configured"}
}
statfs := p.Statfs
if statfs == nil {
statfs = syscall.Statfs
}
var details []string
degraded := false
for _, target := range reduceTargets(targets) {
var stat syscall.Statfs_t
if err := statfs(target.Path, &stat); err != nil {
if target.Optional && errors.Is(err, os.ErrNotExist) {
continue
}
degraded = true
details = append(details, fmt.Sprintf("%s (%s): statfs failed: %v", target.Name, target.Path, err))
continue
}
if stat.Blocks == 0 {
degraded = true
details = append(details, fmt.Sprintf("%s (%s): cannot size filesystem", target.Name, target.Path))
continue
}
used := stat.Blocks - stat.Bavail
pct := float64(used) / float64(stat.Blocks)
status := "ok"
if pct >= p.Threshold {
status = "degraded"
degraded = true
}
details = append(details, fmt.Sprintf("%s (%s): %.0f%% used (limit %.0f%%), %s", target.Name, target.Path, pct*100, p.Threshold*100, status))
}
if len(details) == 0 {
for _, target := range targets {
if !target.Optional {
return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"}
}
}
return health.Result{Status: health.StatusOK, Detail: "no optional disk targets present"}
}
result := health.Result{Status: health.StatusOK, Detail: strings.Join(details, "; ")}
if degraded {
result.Status = health.StatusDegraded
}
return result
}
// --- model ---
// ModelProbe reports whether a model is configured (an active model is
// selected) and reachable (a live network check succeeds). Reachability is a
// narrow function so the probe is deterministic in tests and the composition
// root supplies the real provider probe.
type ModelProbe struct {
ActiveModel func() string
Models func() []string
Reach func(context.Context) error
}
// NewModelProbe returns a model probe. The reachability probe is optional:
// nil means the probe only verifies configuration, not the network.
func NewModelProbe(active func() string, models func() []string, reach func(context.Context) error) *ModelProbe {
return &ModelProbe{ActiveModel: active, Models: models, Reach: reach}
}
func (p *ModelProbe) Name() string { return "model" }
func (p *ModelProbe) Check(ctx context.Context) health.Result {
active := ""
if p.ActiveModel != nil {
active = p.ActiveModel()
}
models := 0
if p.Models != nil {
models = len(p.Models())
}
if active == "" && models == 0 {
return health.Result{Status: health.StatusDegraded, Detail: "no model configured"}
}
if p.Reach == nil {
return health.Result{Status: health.StatusOK, Detail: "model " + active + " configured"}
}
if err := p.Reach(ctx); err != nil {
return health.Result{Status: health.StatusDegraded, Detail: "model unreachable: " + err.Error()}
}
return health.Result{Status: health.StatusOK, Detail: "model " + active + " reachable"}
}
// --- gateway ---
// ChannelState is a per-channel lifecycle fact as the gateway probe sees it.
// It is a projection of the channels.Manager's Status, kept in this package
// so the probe does not depend on the channels package's concrete type.
type ChannelState struct {
ID string
Configured bool
State string
}
// sessionCounter returns the number of currently connected/known sessions.
type sessionCounter func(context.Context) int
// GatewayProbe reports the gateway's running/draining state and its
// connected session count. It consumes injected functions so it stays
// independent of the channels and gateway packages.
type GatewayProbe struct {
Channels func() []ChannelState
CountSessions sessionCounter
}
// NewGatewayProbe returns a gateway probe. Either function may be nil.
func NewGatewayProbe(channels func() []ChannelState, countSessions sessionCounter) *GatewayProbe {
return &GatewayProbe{Channels: channels, CountSessions: countSessions}
}
func (p *GatewayProbe) Name() string { return "gateway" }
func (p *GatewayProbe) Check(ctx context.Context) health.Result {
configChannels := 0
runningChannels := 0
failed := false
if p.Channels != nil {
for _, c := range p.Channels() {
if c.Configured {
configChannels++
}
if c.Configured && c.State == "running" {
runningChannels++
}
if c.Configured && c.State == "failed" {
failed = true
}
}
}
sessions := 0
if p.CountSessions != nil {
sessions = p.CountSessions(ctx)
}
switch {
case configChannels == 0:
return health.Result{
Status: health.StatusDegraded,
Detail: "no gateway channels configured, " + fmt.Sprintf("%d sessions", sessions),
}
case failed:
return health.Result{
Status: health.StatusDegraded,
Detail: fmt.Sprintf("channel failed, %d/%d running, %d sessions", runningChannels, configChannels, sessions),
}
case runningChannels == 0:
return health.Result{
Status: health.StatusDegraded,
Detail: fmt.Sprintf("gateway not running, %d/%d channels up, %d sessions", runningChannels, configChannels, sessions),
}
default:
return health.Result{
Status: health.StatusOK,
Detail: fmt.Sprintf("%d/%d channels running, %d sessions, not draining", runningChannels, configChannels, sessions),
}
}
}