196 lines
6.2 KiB
Go
196 lines
6.2 KiB
Go
package statestore
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/access"
|
|
"github.com/samcharles93/archie-core/internal/domain/alert"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/domain/presence"
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
|
)
|
|
|
|
const (
|
|
// alertSweepInterval is how often the store checks service health and
|
|
// prunes closed alerts.
|
|
alertSweepInterval = time.Minute
|
|
// alertRetention is how long a resolved or dismissed alert is kept.
|
|
alertRetention = 30 * 24 * time.Hour
|
|
// serviceGrace is how long a service may be unhealthy before it is worth
|
|
// an operator's attention: restarts and updates pass through it.
|
|
serviceGrace = 5 * time.Minute
|
|
)
|
|
|
|
// serviceWatch remembers since when each service has been unhealthy.
|
|
type serviceWatch struct {
|
|
since map[string]time.Time
|
|
}
|
|
|
|
// unhealthy returns the services that have been down or degraded for longer
|
|
// than serviceGrace, and forgets those that recovered. A service gone silent
|
|
// counts from its last report, which is stored, so a restart of this store does
|
|
// not restart the grace; one that never reported counts from first sight.
|
|
func (w *serviceWatch) unhealthy(mesh []presence.Service, now time.Time) []string {
|
|
if w.since == nil {
|
|
w.since = map[string]time.Time{}
|
|
}
|
|
var out []string
|
|
for _, service := range mesh {
|
|
if !slices.Contains(presence.Services(), service.Service) {
|
|
continue
|
|
}
|
|
if service.State == presence.StateUp {
|
|
delete(w.since, service.Service)
|
|
continue
|
|
}
|
|
since, ok := w.since[service.Service]
|
|
if !ok {
|
|
since = now
|
|
if service.State == presence.StateDown && !service.ReportedAt.IsZero() {
|
|
since = service.ReportedAt
|
|
}
|
|
w.since[service.Service] = since
|
|
}
|
|
if now.Sub(since) >= serviceGrace {
|
|
out = append(out, service.Service)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func serviceAlert(name string) alert.Input {
|
|
return alert.Input{
|
|
Org: org.DefaultOrgID, Kind: alert.KindServiceUnhealthy, Subject: "service:" + name,
|
|
Severity: alert.SeverityWarning, Condition: true,
|
|
Title: name + " is unhealthy",
|
|
Body: fmt.Sprintf("Down for over %d minutes.", int(serviceGrace.Minutes())),
|
|
Link: "/settings/status",
|
|
}
|
|
}
|
|
|
|
// startSweeps starts the store's periodic work: auto-updating packages, and
|
|
// checking service health and pruning alerts.
|
|
func (b *server) startSweeps(ctx context.Context, deps staterpc.Deps) {
|
|
go b.autoUpdatePackages(ctx, deps.Packages)
|
|
if deps.Alerts != nil {
|
|
go b.sweepAlerts(ctx, deps.Alerts)
|
|
}
|
|
}
|
|
|
|
// sweepAlerts raises and clears service health alerts and prunes closed
|
|
// alerts, until ctx ends. Service health has no transition to hang an alert
|
|
// on, so it is checked here, next to the presence records it reads.
|
|
func (b *server) sweepAlerts(ctx context.Context, store alert.Store) {
|
|
pruner, _ := b.st.(alert.Pruner)
|
|
records, _ := b.st.(interface {
|
|
ListPresence(context.Context) ([]storecontract.Presence, error)
|
|
})
|
|
reporter := alert.Reporter{Raiser: store, Log: b.log, By: "state-store"}
|
|
var watch serviceWatch
|
|
ticker := time.NewTicker(alertSweepInterval)
|
|
defer ticker.Stop()
|
|
for {
|
|
if records != nil {
|
|
b.checkServices(ctx, reporter, records, &watch)
|
|
}
|
|
b.checkPolicies(ctx, reporter, store)
|
|
if pruner != nil {
|
|
if _, err := pruner.PruneAlerts(ctx, time.Now().Add(-alertRetention)); err != nil && ctx.Err() == nil {
|
|
b.log.Warn("prune alerts", "err", err)
|
|
}
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (b *server) checkServices(ctx context.Context, reporter alert.Reporter, records interface {
|
|
ListPresence(context.Context) ([]storecontract.Presence, error)
|
|
}, watch *serviceWatch,
|
|
) {
|
|
list, err := records.ListPresence(ctx)
|
|
if err != nil {
|
|
if ctx.Err() == nil {
|
|
b.log.Warn("service health not checked", "err", err)
|
|
}
|
|
return
|
|
}
|
|
now := time.Now()
|
|
mesh := presence.Mesh(list, now)
|
|
down := watch.unhealthy(mesh, now)
|
|
for _, service := range mesh {
|
|
if !slices.Contains(presence.Services(), service.Service) {
|
|
continue
|
|
}
|
|
// An unhealthy service still inside its grace leaves an open alert
|
|
// alone, so a restart of this store never resolves what it has raised.
|
|
raise := slices.Contains(down, service.Service)
|
|
if raise || service.State == presence.StateUp {
|
|
reporter.Sync(ctx, raise, serviceAlert(service.Service))
|
|
}
|
|
}
|
|
}
|
|
|
|
// checkPolicies raises an alert for every stored policy that does not compile
|
|
// and clears it once the policy validates. An invalid policy keeps denying its
|
|
// level, so the dashboard stays up to repair it: an instance policy asks for
|
|
// action in the system org, any other level warns in its own org. The text is
|
|
// fixed; the compile error is logged once at boot.
|
|
func (b *server) checkPolicies(ctx context.Context, reporter alert.Reporter, store alert.Store) {
|
|
policies, ok := b.st.(access.PolicyStore)
|
|
orgs, hasOrgs := b.st.(org.Repository)
|
|
if !ok || !hasOrgs {
|
|
return
|
|
}
|
|
stored, err := policies.ListPolicies(ctx)
|
|
if err != nil {
|
|
if ctx.Err() == nil {
|
|
b.log.Warn("policies not checked", "err", err)
|
|
}
|
|
return
|
|
}
|
|
invalid := map[alert.Key]bool{}
|
|
for _, problem := range infraaccess.New(stored).Problems() {
|
|
in := policyAlert(problem.Policy)
|
|
invalid[in.Key] = true
|
|
reporter.Raise(ctx, in)
|
|
}
|
|
list, err := orgs.ListOrgs(ctx)
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, o := range list {
|
|
open, err := store.ListAlerts(ctx, o.ID, alert.Filter{})
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for _, a := range open {
|
|
if a.Kind == alert.KindPolicyInvalid && !invalid[a.Key] {
|
|
reporter.Resolve(ctx, a.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func policyAlert(p access.Policy) alert.Input {
|
|
orgID, severity := p.OrgID, alert.SeverityWarning
|
|
if p.Level == access.LevelInstance {
|
|
orgID, severity = org.DefaultOrgID, alert.SeverityAction
|
|
}
|
|
return alert.Input{
|
|
Org: orgID, Kind: alert.KindPolicyInvalid, Subject: "policy:" + p.ID,
|
|
Severity: severity, Condition: true,
|
|
Title: fmt.Sprintf("Access policy %s is invalid", p.ID),
|
|
Body: fmt.Sprintf("The %s-level policy %s does not compile and denies everything at that level.", p.Level, p.ID),
|
|
Link: "/org/access",
|
|
}
|
|
}
|