Merge pull request #846 from samcharles93/feat/835-disk-readiness-probe-watches-only-work-dir-so-a-full-root-fi
Disk readiness probe watches only work_dir, so a full root filesystem goes unreported (archie)
This commit is contained in:
commit
29e5e0130d
3 files changed
+194
-31
No files matched your search
@@ -8,6 +8,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -33,7 +34,7 @@ func (b *boot) setupReadinessProbes() {
|
||||
probes := []health.Probe{
|
||||
readiness.NewStoreProbe(b.stateStore),
|
||||
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
|
||||
readiness.NewDiskProbe(diskProbePath(cfg)),
|
||||
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
|
||||
readiness.NewModelProbe(b.chatModels.ActiveModel, b.chatModels.Models, modelReachProbe(cfg, b.chatModels.ActiveModel)),
|
||||
readiness.NewGatewayProbe(
|
||||
func() []readiness.ChannelState { return channelStates(b.channelManager) },
|
||||
@@ -75,9 +76,20 @@ func sessionCount(ctx context.Context, chat *webui.ChatService) int {
|
||||
return len(snapshot.Sessions)
|
||||
}
|
||||
|
||||
// diskProbePath resolves the filesystem the readiness disk probe inspects:
|
||||
// the store database directory first (the data that must fit on disk), then
|
||||
// the working directory, then the process cwd.
|
||||
// diskProbeTargets returns the filesystems whose capacity matters to the
|
||||
// daemon. Root is always required; /var is useful on conventional Linux
|
||||
// installations but optional so it does not make other environments fail.
|
||||
func diskProbeTargets(cfg config.Config) []readiness.DiskTarget {
|
||||
targets := []readiness.DiskTarget{{Name: "root", Path: "/"}}
|
||||
if info, err := os.Stat("/var"); err == nil && info.IsDir() {
|
||||
targets = append(targets, readiness.DiskTarget{Name: "var", Path: "/var", Optional: true})
|
||||
}
|
||||
targets = append(targets, readiness.DiskTarget{Name: "data", Path: diskProbePath(cfg)})
|
||||
return targets
|
||||
}
|
||||
|
||||
// diskProbePath preserves the database/work/cwd fallback used by the daemon's
|
||||
// disk target: the store database directory first, then work directory, then cwd.
|
||||
func diskProbePath(cfg config.Config) string {
|
||||
switch {
|
||||
case cfg.DBPath != "":
|
||||
|
||||
@@ -12,7 +12,10 @@ package readiness
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -130,42 +133,119 @@ func (p *ConfigProbe) Check(ctx context.Context) health.Result {
|
||||
// requirement of the readiness epic.
|
||||
const DefaultDiskThreshold = 0.90
|
||||
|
||||
// DiskProbe checks the filesystem holding Path is below the used threshold.
|
||||
type DiskProbe struct {
|
||||
Path string
|
||||
Threshold float64
|
||||
// DiskTarget is one filesystem location to measure. Optional targets are
|
||||
// ignored when they disappear (for example, /var on a non-Linux installation).
|
||||
type DiskTarget struct {
|
||||
Name string
|
||||
Path string
|
||||
Optional bool
|
||||
}
|
||||
|
||||
// NewDiskProbe returns a disk probe for the directory Path, degraded when
|
||||
// used usage exceeds the default 90% threshold.
|
||||
// DiskProbe checks each configured filesystem independently and aggregates the
|
||||
// results. Statfs is injectable so callers can test disk boundaries without
|
||||
// filling a filesystem.
|
||||
type DiskProbe struct {
|
||||
// Path is retained for source compatibility with the original single-path
|
||||
// probe. Targets takes precedence when non-empty.
|
||||
Path string
|
||||
Targets []DiskTarget
|
||||
Threshold float64
|
||||
Statfs func(string, *syscall.Statfs_t) error
|
||||
}
|
||||
|
||||
// NewDiskProbe returns a single-path disk probe.
|
||||
func NewDiskProbe(path string) *DiskProbe {
|
||||
return &DiskProbe{Path: path, Threshold: DefaultDiskThreshold}
|
||||
return NewDiskProbeTargets([]DiskTarget{{Name: "work", Path: path}})
|
||||
}
|
||||
|
||||
// NewDiskProbeTargets returns a disk probe for the supplied, stable-ordered targets.
|
||||
func NewDiskProbeTargets(targets []DiskTarget) *DiskProbe {
|
||||
return &DiskProbe{Targets: append([]DiskTarget(nil), targets...), Threshold: DefaultDiskThreshold, Statfs: syscall.Statfs}
|
||||
}
|
||||
|
||||
func (p *DiskProbe) Name() string { return "disk" }
|
||||
|
||||
func (p *DiskProbe) Check(ctx context.Context) health.Result {
|
||||
if p.Path == "" {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"}
|
||||
}
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(p.Path, &stat); err != nil {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "statfs failed: " + err.Error()}
|
||||
}
|
||||
if stat.Blocks == 0 {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "cannot size filesystem"}
|
||||
}
|
||||
// Used fraction uses available (bavail) rather than free (bfree) so a
|
||||
// filesystem with reserved blocks is not reported as safer than it is.
|
||||
used := stat.Blocks - stat.Bavail
|
||||
pct := float64(used) / float64(stat.Blocks)
|
||||
if pct >= p.Threshold {
|
||||
return health.Result{
|
||||
Status: health.StatusDegraded,
|
||||
Detail: fmt.Sprintf("%.0f%% used (limit %.0f%%)", pct*100, p.Threshold*100),
|
||||
// reduceTargets collapses the configured targets to one per path: the first
|
||||
// target seen for a path wins its position, and the result keeps the strictest
|
||||
// requirement any target placed on that path.
|
||||
//
|
||||
// Position is what makes this necessary rather than using a plain set. With
|
||||
// root, an optional /var, then a required data path that resolves to /var, the
|
||||
// optional /var is the target that survives dedup -- and its optional rules
|
||||
// then apply to a path the daemon requires, so a vanished required path is
|
||||
// skipped as "optional absent" and the probe reports OK. Choosing the required
|
||||
// target (and its name) keeps the failure visible and correctly attributed.
|
||||
func reduceTargets(targets []DiskTarget) []DiskTarget {
|
||||
index := make(map[string]int, len(targets))
|
||||
out := make([]DiskTarget, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if target.Path == "" {
|
||||
continue
|
||||
}
|
||||
if at, ok := index[target.Path]; ok {
|
||||
if !target.Optional && out[at].Optional {
|
||||
out[at] = target
|
||||
}
|
||||
continue
|
||||
}
|
||||
index[target.Path] = len(out)
|
||||
out = append(out, target)
|
||||
}
|
||||
return health.Result{Status: health.StatusOK}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *DiskProbe) Check(ctx context.Context) health.Result {
|
||||
_ = ctx
|
||||
targets := p.Targets
|
||||
if len(targets) == 0 && p.Path != "" {
|
||||
targets = []DiskTarget{{Name: "work", Path: p.Path}}
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "no disk targets configured"}
|
||||
}
|
||||
statfs := p.Statfs
|
||||
if statfs == nil {
|
||||
statfs = syscall.Statfs
|
||||
}
|
||||
var details []string
|
||||
degraded := false
|
||||
for _, target := range reduceTargets(targets) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := statfs(target.Path, &stat); err != nil {
|
||||
if target.Optional && errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
degraded = true
|
||||
details = append(details, fmt.Sprintf("%s (%s): statfs failed: %v", target.Name, target.Path, err))
|
||||
continue
|
||||
}
|
||||
if stat.Blocks == 0 {
|
||||
degraded = true
|
||||
details = append(details, fmt.Sprintf("%s (%s): cannot size filesystem", target.Name, target.Path))
|
||||
continue
|
||||
}
|
||||
used := stat.Blocks - stat.Bavail
|
||||
pct := float64(used) / float64(stat.Blocks)
|
||||
status := "ok"
|
||||
if pct >= p.Threshold {
|
||||
status = "degraded"
|
||||
degraded = true
|
||||
}
|
||||
details = append(details, fmt.Sprintf("%s (%s): %.0f%% used (limit %.0f%%), %s", target.Name, target.Path, pct*100, p.Threshold*100, status))
|
||||
}
|
||||
if len(details) == 0 {
|
||||
for _, target := range targets {
|
||||
if !target.Optional {
|
||||
return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"}
|
||||
}
|
||||
}
|
||||
return health.Result{Status: health.StatusOK, Detail: "no optional disk targets present"}
|
||||
}
|
||||
result := health.Result{Status: health.StatusOK, Detail: strings.Join(details, "; ")}
|
||||
if degraded {
|
||||
result.Status = health.StatusDegraded
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- model ---
|
||||
|
||||
@@ -3,6 +3,9 @@ package readiness
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
@@ -196,3 +199,71 @@ func TestGatewayProbe_DegradedWhenNoChannelRunning(t *testing.T) {
|
||||
t.Fatalf("status = %q, want degraded", got.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A required path must not inherit the rules of an earlier optional target on
|
||||
// the same path. Dedup keeps one target per path, and when the optional target
|
||||
// is the one kept, a required path that has gone missing is swallowed as
|
||||
// "optional absent" -- reporting OK while a path the daemon needs is
|
||||
// unavailable. Retaining required status for a path that any target requires
|
||||
// is the fix.
|
||||
func TestDiskProbe_DedupKeepsRequiredWhenOptionalSharesThePath(t *testing.T) {
|
||||
fs := fakeStatfs(map[string]syscall.Statfs_t{
|
||||
"/": {Blocks: 100, Bavail: 100}, // a healthy target keeps the detail list non-empty
|
||||
})
|
||||
// Production orders root first, then the optional /var, then the
|
||||
// config-supplied data path -- which can resolve to /var and collide.
|
||||
p := NewDiskProbeTargets([]DiskTarget{
|
||||
{Name: "root", Path: "/"},
|
||||
{Name: "var", Path: "/var", Optional: true},
|
||||
{Name: "data", Path: "/var"},
|
||||
})
|
||||
p.Statfs = fs.statfs
|
||||
|
||||
got := p.Check(context.Background())
|
||||
if got.Status != health.StatusDegraded {
|
||||
t.Fatalf("status = %q, want degraded: a required path that has vanished must not be reported OK via an optional target's rules (detail: %s)",
|
||||
got.Status, got.Detail)
|
||||
}
|
||||
if !strings.Contains(got.Detail, "data") {
|
||||
t.Fatalf("detail = %q, want the required target named", got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// The inverse: when the kept target is genuinely optional and only optional
|
||||
// targets share the path, an absent path still degrades nothing.
|
||||
func TestDiskProbe_DedupKeepsOptionalWhenNoTargetRequiresThePath(t *testing.T) {
|
||||
fs := fakeStatfs(map[string]syscall.Statfs_t{
|
||||
"/": {Blocks: 100, Bavail: 100},
|
||||
})
|
||||
p := NewDiskProbeTargets([]DiskTarget{
|
||||
{Name: "root", Path: "/"},
|
||||
{Name: "var", Path: "/var", Optional: true},
|
||||
{Name: "var-dup", Path: "/var", Optional: true},
|
||||
})
|
||||
p.Statfs = fs.statfs
|
||||
|
||||
got := p.Check(context.Background())
|
||||
if got.Status != health.StatusOK {
|
||||
t.Fatalf("status = %q, want ok: an optional path absent entirely is not a failure (detail: %s)", got.Status, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeStatfs returns a statfs replacement serving sizes for known paths and
|
||||
// ENOENT for anything else, so a test can drive the disk probe without
|
||||
// filling a real filesystem.
|
||||
func fakeStatfs(sizes map[string]syscall.Statfs_t) struct {
|
||||
statfs func(string, *syscall.Statfs_t) error
|
||||
} {
|
||||
return struct {
|
||||
statfs func(string, *syscall.Statfs_t) error
|
||||
}{
|
||||
statfs: func(path string, out *syscall.Statfs_t) error {
|
||||
stat, ok := sizes[path]
|
||||
if !ok {
|
||||
return os.ErrNotExist
|
||||
}
|
||||
*out = stat
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user