diff --git a/internal/app/archied/readiness.go b/internal/app/archied/readiness.go index 2fc90663a..507efaada 100644 --- a/internal/app/archied/readiness.go +++ b/internal/app/archied/readiness.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "net/http" + "os" "path/filepath" "strings" "time" @@ -33,7 +34,7 @@ func (b *boot) setupReadinessProbes() { probes := []health.Probe{ readiness.NewStoreProbe(b.stateStore), readiness.NewConfigProbe(b.cfgHolder.Get, configuration.Validate), - readiness.NewDiskProbe(diskProbePath(cfg)), + readiness.NewDiskProbeTargets(diskProbeTargets(cfg)), readiness.NewModelProbe(b.chatModels.ActiveModel, b.chatModels.Models, modelReachProbe(cfg, b.chatModels.ActiveModel)), readiness.NewGatewayProbe( func() []readiness.ChannelState { return channelStates(b.channelManager) }, @@ -75,9 +76,20 @@ func sessionCount(ctx context.Context, chat *webui.ChatService) int { return len(snapshot.Sessions) } -// diskProbePath resolves the filesystem the readiness disk probe inspects: -// the store database directory first (the data that must fit on disk), then -// the working directory, then the process cwd. +// diskProbeTargets returns the filesystems whose capacity matters to the +// daemon. Root is always required; /var is useful on conventional Linux +// installations but optional so it does not make other environments fail. +func diskProbeTargets(cfg config.Config) []readiness.DiskTarget { + targets := []readiness.DiskTarget{{Name: "root", Path: "/"}} + if info, err := os.Stat("/var"); err == nil && info.IsDir() { + targets = append(targets, readiness.DiskTarget{Name: "var", Path: "/var", Optional: true}) + } + targets = append(targets, readiness.DiskTarget{Name: "data", Path: diskProbePath(cfg)}) + return targets +} + +// diskProbePath preserves the database/work/cwd fallback used by the daemon's +// disk target: the store database directory first, then work directory, then cwd. func diskProbePath(cfg config.Config) string { switch { case cfg.DBPath != "": diff --git a/internal/infrastructure/readiness/probes.go b/internal/infrastructure/readiness/probes.go index 0104da2f6..2192f3d88 100644 --- a/internal/infrastructure/readiness/probes.go +++ b/internal/infrastructure/readiness/probes.go @@ -12,7 +12,10 @@ package readiness import ( "context" + "errors" "fmt" + "os" + "strings" "syscall" "time" @@ -130,42 +133,119 @@ func (p *ConfigProbe) Check(ctx context.Context) health.Result { // requirement of the readiness epic. const DefaultDiskThreshold = 0.90 -// DiskProbe checks the filesystem holding Path is below the used threshold. -type DiskProbe struct { - Path string - Threshold float64 +// DiskTarget is one filesystem location to measure. Optional targets are +// ignored when they disappear (for example, /var on a non-Linux installation). +type DiskTarget struct { + Name string + Path string + Optional bool } -// NewDiskProbe returns a disk probe for the directory Path, degraded when -// used usage exceeds the default 90% threshold. +// DiskProbe checks each configured filesystem independently and aggregates the +// results. Statfs is injectable so callers can test disk boundaries without +// filling a filesystem. +type DiskProbe struct { + // Path is retained for source compatibility with the original single-path + // probe. Targets takes precedence when non-empty. + Path string + Targets []DiskTarget + Threshold float64 + Statfs func(string, *syscall.Statfs_t) error +} + +// NewDiskProbe returns a single-path disk probe. func NewDiskProbe(path string) *DiskProbe { - return &DiskProbe{Path: path, Threshold: DefaultDiskThreshold} + return NewDiskProbeTargets([]DiskTarget{{Name: "work", Path: path}}) +} + +// NewDiskProbeTargets returns a disk probe for the supplied, stable-ordered targets. +func NewDiskProbeTargets(targets []DiskTarget) *DiskProbe { + return &DiskProbe{Targets: append([]DiskTarget(nil), targets...), Threshold: DefaultDiskThreshold, Statfs: syscall.Statfs} } func (p *DiskProbe) Name() string { return "disk" } -func (p *DiskProbe) Check(ctx context.Context) health.Result { - if p.Path == "" { - return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"} - } - var stat syscall.Statfs_t - if err := syscall.Statfs(p.Path, &stat); err != nil { - return health.Result{Status: health.StatusDegraded, Detail: "statfs failed: " + err.Error()} - } - if stat.Blocks == 0 { - return health.Result{Status: health.StatusDegraded, Detail: "cannot size filesystem"} - } - // Used fraction uses available (bavail) rather than free (bfree) so a - // filesystem with reserved blocks is not reported as safer than it is. - used := stat.Blocks - stat.Bavail - pct := float64(used) / float64(stat.Blocks) - if pct >= p.Threshold { - return health.Result{ - Status: health.StatusDegraded, - Detail: fmt.Sprintf("%.0f%% used (limit %.0f%%)", pct*100, p.Threshold*100), +// reduceTargets collapses the configured targets to one per path: the first +// target seen for a path wins its position, and the result keeps the strictest +// requirement any target placed on that path. +// +// Position is what makes this necessary rather than using a plain set. With +// root, an optional /var, then a required data path that resolves to /var, the +// optional /var is the target that survives dedup -- and its optional rules +// then apply to a path the daemon requires, so a vanished required path is +// skipped as "optional absent" and the probe reports OK. Choosing the required +// target (and its name) keeps the failure visible and correctly attributed. +func reduceTargets(targets []DiskTarget) []DiskTarget { + index := make(map[string]int, len(targets)) + out := make([]DiskTarget, 0, len(targets)) + for _, target := range targets { + if target.Path == "" { + continue } + if at, ok := index[target.Path]; ok { + if !target.Optional && out[at].Optional { + out[at] = target + } + continue + } + index[target.Path] = len(out) + out = append(out, target) } - return health.Result{Status: health.StatusOK} + return out +} + +func (p *DiskProbe) Check(ctx context.Context) health.Result { + _ = ctx + targets := p.Targets + if len(targets) == 0 && p.Path != "" { + targets = []DiskTarget{{Name: "work", Path: p.Path}} + } + if len(targets) == 0 { + return health.Result{Status: health.StatusDegraded, Detail: "no disk targets configured"} + } + statfs := p.Statfs + if statfs == nil { + statfs = syscall.Statfs + } + var details []string + degraded := false + for _, target := range reduceTargets(targets) { + var stat syscall.Statfs_t + if err := statfs(target.Path, &stat); err != nil { + if target.Optional && errors.Is(err, os.ErrNotExist) { + continue + } + degraded = true + details = append(details, fmt.Sprintf("%s (%s): statfs failed: %v", target.Name, target.Path, err)) + continue + } + if stat.Blocks == 0 { + degraded = true + details = append(details, fmt.Sprintf("%s (%s): cannot size filesystem", target.Name, target.Path)) + continue + } + used := stat.Blocks - stat.Bavail + pct := float64(used) / float64(stat.Blocks) + status := "ok" + if pct >= p.Threshold { + status = "degraded" + degraded = true + } + details = append(details, fmt.Sprintf("%s (%s): %.0f%% used (limit %.0f%%), %s", target.Name, target.Path, pct*100, p.Threshold*100, status)) + } + if len(details) == 0 { + for _, target := range targets { + if !target.Optional { + return health.Result{Status: health.StatusDegraded, Detail: "no disk path configured"} + } + } + return health.Result{Status: health.StatusOK, Detail: "no optional disk targets present"} + } + result := health.Result{Status: health.StatusOK, Detail: strings.Join(details, "; ")} + if degraded { + result.Status = health.StatusDegraded + } + return result } // --- model --- diff --git a/internal/infrastructure/readiness/probes_test.go b/internal/infrastructure/readiness/probes_test.go index 2e348b9ef..be2560642 100644 --- a/internal/infrastructure/readiness/probes_test.go +++ b/internal/infrastructure/readiness/probes_test.go @@ -3,6 +3,9 @@ package readiness import ( "context" "errors" + "os" + "strings" + "syscall" "testing" "github.com/samcharles93/archie-core/internal/config" @@ -196,3 +199,71 @@ func TestGatewayProbe_DegradedWhenNoChannelRunning(t *testing.T) { t.Fatalf("status = %q, want degraded", got.Status) } } + +// A required path must not inherit the rules of an earlier optional target on +// the same path. Dedup keeps one target per path, and when the optional target +// is the one kept, a required path that has gone missing is swallowed as +// "optional absent" -- reporting OK while a path the daemon needs is +// unavailable. Retaining required status for a path that any target requires +// is the fix. +func TestDiskProbe_DedupKeepsRequiredWhenOptionalSharesThePath(t *testing.T) { + fs := fakeStatfs(map[string]syscall.Statfs_t{ + "/": {Blocks: 100, Bavail: 100}, // a healthy target keeps the detail list non-empty + }) + // Production orders root first, then the optional /var, then the + // config-supplied data path -- which can resolve to /var and collide. + p := NewDiskProbeTargets([]DiskTarget{ + {Name: "root", Path: "/"}, + {Name: "var", Path: "/var", Optional: true}, + {Name: "data", Path: "/var"}, + }) + p.Statfs = fs.statfs + + got := p.Check(context.Background()) + if got.Status != health.StatusDegraded { + t.Fatalf("status = %q, want degraded: a required path that has vanished must not be reported OK via an optional target's rules (detail: %s)", + got.Status, got.Detail) + } + if !strings.Contains(got.Detail, "data") { + t.Fatalf("detail = %q, want the required target named", got.Detail) + } +} + +// The inverse: when the kept target is genuinely optional and only optional +// targets share the path, an absent path still degrades nothing. +func TestDiskProbe_DedupKeepsOptionalWhenNoTargetRequiresThePath(t *testing.T) { + fs := fakeStatfs(map[string]syscall.Statfs_t{ + "/": {Blocks: 100, Bavail: 100}, + }) + p := NewDiskProbeTargets([]DiskTarget{ + {Name: "root", Path: "/"}, + {Name: "var", Path: "/var", Optional: true}, + {Name: "var-dup", Path: "/var", Optional: true}, + }) + p.Statfs = fs.statfs + + got := p.Check(context.Background()) + if got.Status != health.StatusOK { + t.Fatalf("status = %q, want ok: an optional path absent entirely is not a failure (detail: %s)", got.Status, got.Detail) + } +} + +// fakeStatfs returns a statfs replacement serving sizes for known paths and +// ENOENT for anything else, so a test can drive the disk probe without +// filling a real filesystem. +func fakeStatfs(sizes map[string]syscall.Statfs_t) struct { + statfs func(string, *syscall.Statfs_t) error +} { + return struct { + statfs func(string, *syscall.Statfs_t) error + }{ + statfs: func(path string, out *syscall.Statfs_t) error { + stat, ok := sizes[path] + if !ok { + return os.ErrNotExist + } + *out = stat + return nil + }, + } +}