Merge pull request #846 from samcharles93/feat/835-disk-readiness-probe-watches-only-work-dir-so-a-full-root-fi

Disk readiness probe watches only work_dir, so a full root filesystem goes unreported (archie)
This commit is contained in:
sam authored and GitHub committed 2026-09-17 22:39:15 +10:00
commit 29e5e0130d
3 files changed
+194 -31

No files matched your search

+16 -4
View File
@@ -8,6 +8,7 @@ import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
@@ -33,7 +34,7 @@ func (b *boot) setupReadinessProbes() {
probes := []health.Probe{
readiness.NewStoreProbe(b.stateStore),
readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate),
readiness.NewDiskProbe(diskProbePath(cfg)),
readiness.NewDiskProbeTargets(diskProbeTargets(cfg)),
readiness.NewModelProbe(b.chatModels.ActiveModel, b.chatModels.Models, modelReachProbe(cfg, b.chatModels.ActiveModel)),
readiness.NewGatewayProbe(
func() []readiness.ChannelState { return channelStates(b.channelManager) },
@@ -75,9 +76,20 @@ func sessionCount(ctx context.Context, chat *webui.ChatService) int {
return len(snapshot.Sessions)
}
// diskProbePath resolves the filesystem the readiness disk probe inspects:
// the store database directory first (the data that must fit on disk), then
// the working directory, then the process cwd.
// diskProbeTargets returns the filesystems whose capacity matters to the
// daemon. Root is always required; /var is useful on conventional Linux
// installations but optional so it does not make other environments fail.
func diskProbeTargets(cfg config.Config) []readiness.DiskTarget {
targets := []readiness.DiskTarget{{Name: "root", Path: "/"}}
if info, err := os.Stat("/var"); err == nil && info.IsDir() {
targets = append(targets, readiness.DiskTarget{Name: "var", Path: "/var", Optional: true})
}
targets = append(targets, readiness.DiskTarget{Name: "data", Path: diskProbePath(cfg)})
return targets
}
// diskProbePath preserves the database/work/cwd fallback used by the daemon's
// disk target: the store database directory first, then work directory, then cwd.
func diskProbePath(cfg config.Config) string {
switch {
case cfg.DBPath != "":
+107 -27
View File
@@ -12,7 +12,10 @@ package readiness
import (
"context"
"errors"
"fmt"
"os"
"strings"
"syscall"
"time"
@@ -130,42 +133,119 @@ func (p *ConfigProbe) Check(ctx context.Context) health.Result {
// requirement of the readiness epic.
const DefaultDiskThreshold = 0.90
// DiskProbe checks the filesystem holding Path is below the used threshold.
type DiskProbe struct {
Path string
Threshold float64
// DiskTarget is one filesystem location to measure. Optional targets are
// ignored when they disappear (for example, /var on a non-Linux installation).
type DiskTarget struct {
Name string
Path string
Optional bool
}
// NewDiskProbe returns a disk probe for the directory Path, degraded when
// used usage exceeds the default 90% threshold.
// DiskProbe checks each configured filesystem independently and aggregates the
// results. Statfs is injectable so callers can test disk boundaries without
// filling a filesystem.
type DiskProbe struct {
// Path is retained for source compatibility with the original single-path
// probe. Targets takes precedence when non-empty.
Path string
Targets []DiskTarget
Threshold float64
Statfs func(string, *syscall.Statfs_t) error
}
// NewDiskProbe returns a single-path disk probe.
func NewDiskProbe(path string) *DiskProbe {
return &DiskProbe{Path: path, Threshold: DefaultDiskThreshold}
return NewDiskProbeTargets([]DiskTarget{{Name: "work", Path: path}})
}
// NewDiskProbeTargets returns a disk probe for the supplied, stable-ordered targets.
func NewDiskProbeTargets(targets []DiskTarget) *DiskProbe {
return &DiskProbe{Targets: append([]DiskTarget(nil), targets...), Threshold: DefaultDiskThreshold, Statfs: syscall.Statfs}
}
func (p *DiskProbe) Name() string { return "disk" }
func (p *DiskProbe) Check(ctx context.Context) health.Result {
if p.Path == "" {
return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"}
}
var stat syscall.Statfs_t
if err := syscall.Statfs(p.Path, &stat); err != nil {
return health.Result{Status: health.StatusDegraded, Detail: "statfs failed: " + err.Error()}
}
if stat.Blocks == 0 {
return health.Result{Status: health.StatusDegraded, Detail: "cannot size filesystem"}
}
// Used fraction uses available (bavail) rather than free (bfree) so a
// filesystem with reserved blocks is not reported as safer than it is.
used := stat.Blocks - stat.Bavail
pct := float64(used) / float64(stat.Blocks)
if pct >= p.Threshold {
return health.Result{
Status: health.StatusDegraded,
Detail: fmt.Sprintf("%.0f%% used (limit %.0f%%)", pct*100, p.Threshold*100),
// reduceTargets collapses the configured targets to one per path: the first
// target seen for a path wins its position, and the result keeps the strictest
// requirement any target placed on that path.
//
// Position is what makes this necessary rather than using a plain set. With
// root, an optional /var, then a required data path that resolves to /var, the
// optional /var is the target that survives dedup -- and its optional rules
// then apply to a path the daemon requires, so a vanished required path is
// skipped as "optional absent" and the probe reports OK. Choosing the required
// target (and its name) keeps the failure visible and correctly attributed.
func reduceTargets(targets []DiskTarget) []DiskTarget {
index := make(map[string]int, len(targets))
out := make([]DiskTarget, 0, len(targets))
for _, target := range targets {
if target.Path == "" {
continue
}
if at, ok := index[target.Path]; ok {
if !target.Optional && out[at].Optional {
out[at] = target
}
continue
}
index[target.Path] = len(out)
out = append(out, target)
}
return health.Result{Status: health.StatusOK}
return out
}
func (p *DiskProbe) Check(ctx context.Context) health.Result {
_ = ctx
targets := p.Targets
if len(targets) == 0 && p.Path != "" {
targets = []DiskTarget{{Name: "work", Path: p.Path}}
}
if len(targets) == 0 {
return health.Result{Status: health.StatusDegraded, Detail: "no disk targets configured"}
}
statfs := p.Statfs
if statfs == nil {
statfs = syscall.Statfs
}
var details []string
degraded := false
for _, target := range reduceTargets(targets) {
var stat syscall.Statfs_t
if err := statfs(target.Path, &stat); err != nil {
if target.Optional && errors.Is(err, os.ErrNotExist) {
continue
}
degraded = true
details = append(details, fmt.Sprintf("%s (%s): statfs failed: %v", target.Name, target.Path, err))
continue
}
if stat.Blocks == 0 {
degraded = true
details = append(details, fmt.Sprintf("%s (%s): cannot size filesystem", target.Name, target.Path))
continue
}
used := stat.Blocks - stat.Bavail
pct := float64(used) / float64(stat.Blocks)
status := "ok"
if pct >= p.Threshold {
status = "degraded"
degraded = true
}
details = append(details, fmt.Sprintf("%s (%s): %.0f%% used (limit %.0f%%), %s", target.Name, target.Path, pct*100, p.Threshold*100, status))
}
if len(details) == 0 {
for _, target := range targets {
if !target.Optional {
return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"}
}
}
return health.Result{Status: health.StatusOK, Detail: "no optional disk targets present"}
}
result := health.Result{Status: health.StatusOK, Detail: strings.Join(details, "; ")}
if degraded {
result.Status = health.StatusDegraded
}
return result
}
// --- model ---
@@ -3,6 +3,9 @@ package readiness
import (
"context"
"errors"
"os"
"strings"
"syscall"
"testing"
"github.com/samcharles93/archie-core/internal/config"
@@ -196,3 +199,71 @@ func TestGatewayProbe_DegradedWhenNoChannelRunning(t *testing.T) {
t.Fatalf("status = %q, want degraded", got.Status)
}
}
// A required path must not inherit the rules of an earlier optional target on
// the same path. Dedup keeps one target per path, and when the optional target
// is the one kept, a required path that has gone missing is swallowed as
// "optional absent" -- reporting OK while a path the daemon needs is
// unavailable. Retaining required status for a path that any target requires
// is the fix.
func TestDiskProbe_DedupKeepsRequiredWhenOptionalSharesThePath(t *testing.T) {
fs := fakeStatfs(map[string]syscall.Statfs_t{
"/": {Blocks: 100, Bavail: 100}, // a healthy target keeps the detail list non-empty
})
// Production orders root first, then the optional /var, then the
// config-supplied data path -- which can resolve to /var and collide.
p := NewDiskProbeTargets([]DiskTarget{
{Name: "root", Path: "/"},
{Name: "var", Path: "/var", Optional: true},
{Name: "data", Path: "/var"},
})
p.Statfs = fs.statfs
got := p.Check(context.Background())
if got.Status != health.StatusDegraded {
t.Fatalf("status = %q, want degraded: a required path that has vanished must not be reported OK via an optional target's rules (detail: %s)",
got.Status, got.Detail)
}
if !strings.Contains(got.Detail, "data") {
t.Fatalf("detail = %q, want the required target named", got.Detail)
}
}
// The inverse: when the kept target is genuinely optional and only optional
// targets share the path, an absent path still degrades nothing.
func TestDiskProbe_DedupKeepsOptionalWhenNoTargetRequiresThePath(t *testing.T) {
fs := fakeStatfs(map[string]syscall.Statfs_t{
"/": {Blocks: 100, Bavail: 100},
})
p := NewDiskProbeTargets([]DiskTarget{
{Name: "root", Path: "/"},
{Name: "var", Path: "/var", Optional: true},
{Name: "var-dup", Path: "/var", Optional: true},
})
p.Statfs = fs.statfs
got := p.Check(context.Background())
if got.Status != health.StatusOK {
t.Fatalf("status = %q, want ok: an optional path absent entirely is not a failure (detail: %s)", got.Status, got.Detail)
}
}
// fakeStatfs returns a statfs replacement serving sizes for known paths and
// ENOENT for anything else, so a test can drive the disk probe without
// filling a real filesystem.
func fakeStatfs(sizes map[string]syscall.Statfs_t) struct {
statfs func(string, *syscall.Statfs_t) error
} {
return struct {
statfs func(string, *syscall.Statfs_t) error
}{
statfs: func(path string, out *syscall.Statfs_t) error {
stat, ok := sizes[path]
if !ok {
return os.ErrNotExist
}
*out = stat
return nil
},
}
}