Files
archie/internal/app/statestore/alerts.go
T
sam cc5789eb5f feat(scheduling): schedules and runs in the State Store with grid timing and run history
Schedules leave the schedules control-plane resource for State Store tables
behind a ScheduleService contract. A run is claimed per schedule slot, missed
slots fire once, the next run stays on the schedule's grid, cron is read in the
schedule-settings time zone, and failed runs are recorded and advance. Three
failures in a row raise a schedule.failing alert. A fired run records a
schedule-fired capture. Run history is pruned by age and count.

Removes the chat and spawn-task runners, the dangerous-payload guard and the
old schedules settings page.
2026-10-26 00:49:24 +11:00

198 lines
6.3 KiB
Go

package statestore
import (
"context"
"fmt"
"slices"
"time"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
)
const (
// alertSweepInterval is how often the store checks service health and
// prunes closed alerts.
alertSweepInterval = time.Minute
// alertRetention is how long a resolved or dismissed alert is kept.
alertRetention = 30 * 24 * time.Hour
// serviceGrace is how long a service may be unhealthy before it is worth
// an operator's attention: restarts and updates pass through it.
serviceGrace = 5 * time.Minute
)
// serviceWatch remembers since when each service has been unhealthy.
type serviceWatch struct {
since map[string]time.Time
}
// unhealthy returns the services that have been down or degraded for longer
// than serviceGrace, and forgets those that recovered. A service gone silent
// counts from its last report, which is stored, so a restart of this store does
// not restart the grace; one that never reported counts from first sight.
func (w *serviceWatch) unhealthy(mesh []presence.Service, now time.Time) []string {
if w.since == nil {
w.since = map[string]time.Time{}
}
var out []string
for _, service := range mesh {
if !slices.Contains(presence.Services(), service.Service) {
continue
}
if service.State == presence.StateUp {
delete(w.since, service.Service)
continue
}
since, ok := w.since[service.Service]
if !ok {
since = now
if service.State == presence.StateDown && !service.ReportedAt.IsZero() {
since = service.ReportedAt
}
w.since[service.Service] = since
}
if now.Sub(since) >= serviceGrace {
out = append(out, service.Service)
}
}
return out
}
func serviceAlert(name string) alert.Input {
return alert.Input{
Org: org.DefaultOrgID, Kind: alert.KindServiceUnhealthy, Subject: "service:" + name,
Severity: alert.SeverityWarning, Condition: true,
Title: name + " is unhealthy",
Body: fmt.Sprintf("Down for over %d minutes.", int(serviceGrace.Minutes())),
Link: "/settings/status",
}
}
// startSweeps starts the store's periodic work: auto-updating packages,
// checking service health and pruning alerts, and applying audit retention.
func (b *server) startSweeps(ctx context.Context, deps staterpc.Deps) {
go b.autoUpdatePackages(ctx, deps.Packages)
go b.sweepAuditRetention(ctx)
go b.sweepScheduleRuns(ctx, deps)
if deps.Alerts != nil {
go b.sweepAlerts(ctx, deps.Alerts)
}
}
// sweepAlerts raises and clears service health alerts and prunes closed
// alerts, until ctx ends. Service health has no transition to hang an alert
// on, so it is checked here, next to the presence records it reads.
func (b *server) sweepAlerts(ctx context.Context, store alert.Store) {
pruner, _ := b.st.(alert.Pruner)
records, _ := b.st.(interface {
ListPresence(context.Context) ([]storecontract.Presence, error)
})
reporter := alert.Reporter{Raiser: store, Log: b.log, By: "state-store"}
var watch serviceWatch
ticker := time.NewTicker(alertSweepInterval)
defer ticker.Stop()
for {
if records != nil {
b.checkServices(ctx, reporter, records, &watch)
}
b.checkPolicies(ctx, reporter, store)
if pruner != nil {
if _, err := pruner.PruneAlerts(ctx, time.Now().Add(-alertRetention)); err != nil && ctx.Err() == nil {
b.log.WarnContext(ctx, "prune alerts", "err", err)
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (b *server) checkServices(ctx context.Context, reporter alert.Reporter, records interface {
ListPresence(context.Context) ([]storecontract.Presence, error)
}, watch *serviceWatch,
) {
list, err := records.ListPresence(ctx)
if err != nil {
if ctx.Err() == nil {
b.log.WarnContext(ctx, "service health not checked", "err", err)
}
return
}
now := time.Now()
mesh := presence.Mesh(list, now)
down := watch.unhealthy(mesh, now)
for _, service := range mesh {
if !slices.Contains(presence.Services(), service.Service) {
continue
}
// An unhealthy service still inside its grace leaves an open alert
// alone, so a restart of this store never resolves what it has raised.
raise := slices.Contains(down, service.Service)
if raise || service.State == presence.StateUp {
reporter.Sync(ctx, raise, serviceAlert(service.Service))
}
}
}
// checkPolicies raises an alert for every stored policy that does not compile
// and clears it once the policy validates. An invalid policy keeps denying its
// level, so the dashboard stays up to repair it: an instance policy asks for
// action in the system org, any other level warns in its own org. The text is
// fixed; the compile error is logged once at boot.
func (b *server) checkPolicies(ctx context.Context, reporter alert.Reporter, store alert.Store) {
policies, ok := b.st.(access.PolicyStore)
orgs, hasOrgs := b.st.(org.Repository)
if !ok || !hasOrgs {
return
}
stored, err := policies.ListPolicies(ctx)
if err != nil {
if ctx.Err() == nil {
b.log.WarnContext(ctx, "policies not checked", "err", err)
}
return
}
invalid := map[alert.Key]bool{}
for _, problem := range infraaccess.New(stored).Problems() {
in := policyAlert(problem.Policy)
invalid[in.Key] = true
reporter.Raise(ctx, in)
}
list, err := orgs.ListOrgs(ctx)
if err != nil {
return
}
for _, o := range list {
open, err := store.ListAlerts(ctx, o.ID, alert.Filter{})
if err != nil {
continue
}
for _, a := range open {
if a.Kind == alert.KindPolicyInvalid && !invalid[a.Key] {
reporter.Resolve(ctx, a.Key)
}
}
}
}
func policyAlert(p access.Policy) alert.Input {
orgID, severity := p.OrgID, alert.SeverityWarning
if p.Level == access.LevelInstance {
orgID, severity = org.DefaultOrgID, alert.SeverityAction
}
return alert.Input{
Org: orgID, Kind: alert.KindPolicyInvalid, Subject: "policy:" + p.ID,
Severity: severity, Condition: true,
Title: fmt.Sprintf("Access policy %s is invalid", p.ID),
Body: fmt.Sprintf("The %s-level policy %s does not compile and denies everything at that level.", p.Level, p.ID),
Link: "/org/access",
}
}