Files
archie/internal/domain/alert/alert.go
T

281 lines
8.7 KiB
Go

// Package alert is the operator's list of things that need a human. The State
// Store owns the rows; services raise and clear alerts through its contract and
// never read the table.
//
// An alert is one row per (org, kind, subject). A condition alert is backed by
// something that can stop being true, so whoever detects it resolves it when
// it clears. An event alert records something that happened and stays until
// an operator dismisses it.
//
// Titles and bodies are shown to every member of the org and sent to the
// notify webhook. A raiser writes them from fixed text and ids, never from an
// error string, a command line or a credential.
package alert
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/samcharles93/archie-core/internal/domain/org"
)
// Severity orders how loudly an alert asks for attention.
type Severity string
const (
SeverityInfo Severity = "info"
SeverityWarning Severity = "warning"
SeverityAction Severity = "action"
)
// Rank orders severities, highest first needing the most attention. An unknown
// severity ranks below info.
func (s Severity) Rank() int {
switch s {
case SeverityAction:
return 3
case SeverityWarning:
return 2
case SeverityInfo:
return 1
default:
return 0
}
}
// Valid reports whether s is a known severity.
func (s Severity) Valid() bool { return s.Rank() > 0 }
// State is where an alert is in its life.
type State string
const (
StateOpen State = "open"
StateResolved State = "resolved"
StateDismissed State = "dismissed"
)
// Conditions and events the in-tree raisers use. The kind is part of the
// dedup key, so renaming one orphans its open alerts.
const (
KindTaskNeedsHuman = "task.needs_human"
KindTaskFailed = "task.failed"
KindModelAlias = "model.alias"
KindProviderNoKey = "provider.no_credential"
KindToolNoBinding = "tool.no_credential"
KindCredentialInvalid = "credential.rejected"
KindServiceUnhealthy = "service.unhealthy"
KindReleaseAvailable = "release.available"
KindPolicyInvalid = "policy.invalid"
)
// Limits on what a raiser may store.
const (
maxKindLen = 64
maxSubjectLen = 200
maxTitleLen = 200
maxBodyLen = 2000
maxLinkLen = 500
)
// ErrInvalid is returned for an alert the store would refuse to hold.
var ErrInvalid = errors.New("alert: invalid")
// Key is what makes two raises the same alert.
type Key struct {
Org org.OrgID
Kind string
Subject string
}
// Input is one raise.
type Input struct {
Key
Severity Severity
Title string
Body string
// Link is the dashboard path that fixes it, such as /tasks/12.
Link string
// Condition is true when the raiser resolves the alert itself once the
// condition clears. An event alert (false) stays until dismissed.
Condition bool
}
// Validate refuses an input the store cannot hold.
func (in Input) Validate() error {
switch {
case in.Org == "":
return fmt.Errorf("%w: org is required", ErrInvalid)
case in.Kind == "" || len(in.Kind) > maxKindLen:
return fmt.Errorf("%w: kind is required and at most %d bytes", ErrInvalid, maxKindLen)
case in.Subject == "" || len(in.Subject) > maxSubjectLen:
return fmt.Errorf("%w: subject is required and at most %d bytes", ErrInvalid, maxSubjectLen)
case !in.Severity.Valid():
return fmt.Errorf("%w: severity %q", ErrInvalid, in.Severity)
case strings.TrimSpace(in.Title) == "" || len(in.Title) > maxTitleLen:
return fmt.Errorf("%w: title is required and at most %d bytes", ErrInvalid, maxTitleLen)
case len(in.Body) > maxBodyLen:
return fmt.Errorf("%w: body is at most %d bytes", ErrInvalid, maxBodyLen)
case len(in.Link) > maxLinkLen || (in.Link != "" && !strings.HasPrefix(in.Link, "/")):
return fmt.Errorf("%w: link must be a dashboard path of at most %d bytes", ErrInvalid, maxLinkLen)
}
return nil
}
// Alert is one row.
type Alert struct {
ID int64
Key
Severity Severity
Title string
Body string
Link string
Condition bool
State State
FirstRaised time.Time
LastSeen time.Time
Count int64
// ClosedBy and ClosedAt say who resolved or dismissed the alert and when.
ClosedBy string
ClosedAt time.Time
// Deliveries are the outbound notices this alert owed, newest first.
Deliveries []Delivery
}
// Apply is the alert after a raise. prev is the existing row, nil when there
// is none. opened reports that the alert became open: it is new, or it was
// resolved, or it was an event alert that had been dismissed. An open alert
// raised again only bumps its count and last seen. A dismissed condition
// alert is snoozed: it stays dismissed until the condition clears and recurs.
func Apply(prev *Alert, in Input, now time.Time) (next Alert, opened bool) {
fresh := Alert{
Key: in.Key, Severity: in.Severity, Title: in.Title, Body: in.Body, Link: in.Link,
Condition: in.Condition, State: StateOpen, FirstRaised: now, LastSeen: now, Count: 1,
}
if prev == nil {
return fresh, true
}
next = *prev
next.Severity, next.Title, next.Body, next.Link, next.Condition = in.Severity, in.Title, in.Body, in.Link, in.Condition
next.LastSeen = now
switch {
case prev.State == StateOpen:
next.Count++
return next, false
case prev.State == StateDismissed && prev.Condition && in.Condition:
return next, false
default:
fresh.ID, fresh.Deliveries = prev.ID, prev.Deliveries
return fresh, true
}
}
// Resolve is the alert after its condition cleared. changed reports that it
// moved to resolved. Only a condition alert has a condition to clear: an
// event alert stays until dismissed, and a resolved alert stays as it is. A
// snoozed (dismissed) condition alert resolves, so it can come back if the
// condition recurs.
func Resolve(prev Alert, by string, now time.Time) (next Alert, changed bool) {
if !prev.Condition || prev.State == StateResolved {
return prev, false
}
prev.State, prev.ClosedBy, prev.ClosedAt = StateResolved, by, now
return prev, true
}
// Dismiss is the alert after an operator dismissed it. changed reports that it
// moved to dismissed; only an open alert can be dismissed.
func Dismiss(prev Alert, by string, now time.Time) (next Alert, changed bool) {
if prev.State != StateOpen {
return prev, false
}
prev.State, prev.ClosedBy, prev.ClosedAt = StateDismissed, by, now
return prev, true
}
// Filter selects what List returns.
type Filter struct {
// States to include; empty means open only.
States []State
Limit int
}
// Store is the State Store's alert surface.
type Store interface {
Raiser
// Dismiss dismisses an open alert in org.
DismissAlert(ctx context.Context, orgID org.OrgID, id int64, by string) error
// List returns an org's alerts, highest severity then newest first.
ListAlerts(ctx context.Context, orgID org.OrgID, filter Filter) ([]Alert, error)
Deliverer
}
// Pruner removes alerts that closed before a retention cutoff, with their
// deliveries. The State Store runs it; it is not part of the contract.
type Pruner interface {
PruneAlerts(ctx context.Context, cutoff time.Time) (int64, error)
}
// Raiser is what a service needs to raise and clear alerts.
type Raiser interface {
// RaiseAlert stores in, idempotently: see Apply.
RaiseAlert(ctx context.Context, in Input) (Alert, error)
// ResolveAlert clears a condition alert; no alert under key is not an
// error.
ResolveAlert(ctx context.Context, key Key, by string) error
}
// Reporter raises and clears alerts for a service. A failure is logged and
// left for the next check: reporting never blocks or fails the caller's work.
// The zero Reporter, and one with no Raiser, does nothing.
type Reporter struct {
Raiser Raiser
Log *slog.Logger
// By names the resolver recorded on alerts this reporter clears.
By string
}
// Raise stores in.
func (r Reporter) Raise(ctx context.Context, in Input) {
if r.Raiser == nil {
return
}
if _, err := r.Raiser.RaiseAlert(ctx, in); err != nil {
r.warn("alert not raised", in.Key, err)
}
}
// Resolve clears the alert under key.
func (r Reporter) Resolve(ctx context.Context, key Key) {
if r.Raiser == nil {
return
}
if err := r.Raiser.ResolveAlert(ctx, key, r.By); err != nil {
r.warn("alert not resolved", key, err)
}
}
// Sync raises in while active holds and resolves it once it does not. It is
// the whole body of a periodic check for one condition.
func (r Reporter) Sync(ctx context.Context, active bool, in Input) {
if active {
r.Raise(ctx, in)
return
}
r.Resolve(ctx, in.Key)
}
func (r Reporter) warn(msg string, key Key, err error) {
if r.Log != nil {
r.Log.Warn(msg, "org", key.Org, "kind", key.Kind, "subject", key.Subject, "err", err)
}
}
// ErrNotFound is returned for an alert or delivery that does not exist, or
// that belongs to another org.
var ErrNotFound = errors.New("alert: not found")