281 lines
8.7 KiB
Go
281 lines
8.7 KiB
Go
// Package alert is the operator's list of things that need a human. The State
|
|
// Store owns the rows; services raise and clear alerts through its contract and
|
|
// never read the table.
|
|
//
|
|
// An alert is one row per (org, kind, subject). A condition alert is backed by
|
|
// something that can stop being true, so whoever detects it resolves it when
|
|
// it clears. An event alert records something that happened and stays until
|
|
// an operator dismisses it.
|
|
//
|
|
// Titles and bodies are shown to every member of the org and sent to the
|
|
// notify webhook. A raiser writes them from fixed text and ids, never from an
|
|
// error string, a command line or a credential.
|
|
package alert
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
)
|
|
|
|
// Severity orders how loudly an alert asks for attention.
|
|
type Severity string
|
|
|
|
const (
|
|
SeverityInfo Severity = "info"
|
|
SeverityWarning Severity = "warning"
|
|
SeverityAction Severity = "action"
|
|
)
|
|
|
|
// Rank orders severities, highest first needing the most attention. An unknown
|
|
// severity ranks below info.
|
|
func (s Severity) Rank() int {
|
|
switch s {
|
|
case SeverityAction:
|
|
return 3
|
|
case SeverityWarning:
|
|
return 2
|
|
case SeverityInfo:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// Valid reports whether s is a known severity.
|
|
func (s Severity) Valid() bool { return s.Rank() > 0 }
|
|
|
|
// State is where an alert is in its life.
|
|
type State string
|
|
|
|
const (
|
|
StateOpen State = "open"
|
|
StateResolved State = "resolved"
|
|
StateDismissed State = "dismissed"
|
|
)
|
|
|
|
// Conditions and events the in-tree raisers use. The kind is part of the
|
|
// dedup key, so renaming one orphans its open alerts.
|
|
const (
|
|
KindTaskNeedsHuman = "task.needs_human"
|
|
KindTaskFailed = "task.failed"
|
|
KindModelAlias = "model.alias"
|
|
KindProviderNoKey = "provider.no_credential"
|
|
KindToolNoBinding = "tool.no_credential"
|
|
KindCredentialInvalid = "credential.rejected"
|
|
KindServiceUnhealthy = "service.unhealthy"
|
|
KindReleaseAvailable = "release.available"
|
|
KindPolicyInvalid = "policy.invalid"
|
|
)
|
|
|
|
// Limits on what a raiser may store.
|
|
const (
|
|
maxKindLen = 64
|
|
maxSubjectLen = 200
|
|
maxTitleLen = 200
|
|
maxBodyLen = 2000
|
|
maxLinkLen = 500
|
|
)
|
|
|
|
// ErrInvalid is returned for an alert the store would refuse to hold.
|
|
var ErrInvalid = errors.New("alert: invalid")
|
|
|
|
// Key is what makes two raises the same alert.
|
|
type Key struct {
|
|
Org org.OrgID
|
|
Kind string
|
|
Subject string
|
|
}
|
|
|
|
// Input is one raise.
|
|
type Input struct {
|
|
Key
|
|
Severity Severity
|
|
Title string
|
|
Body string
|
|
// Link is the dashboard path that fixes it, such as /tasks/12.
|
|
Link string
|
|
// Condition is true when the raiser resolves the alert itself once the
|
|
// condition clears. An event alert (false) stays until dismissed.
|
|
Condition bool
|
|
}
|
|
|
|
// Validate refuses an input the store cannot hold.
|
|
func (in Input) Validate() error {
|
|
switch {
|
|
case in.Org == "":
|
|
return fmt.Errorf("%w: org is required", ErrInvalid)
|
|
case in.Kind == "" || len(in.Kind) > maxKindLen:
|
|
return fmt.Errorf("%w: kind is required and at most %d bytes", ErrInvalid, maxKindLen)
|
|
case in.Subject == "" || len(in.Subject) > maxSubjectLen:
|
|
return fmt.Errorf("%w: subject is required and at most %d bytes", ErrInvalid, maxSubjectLen)
|
|
case !in.Severity.Valid():
|
|
return fmt.Errorf("%w: severity %q", ErrInvalid, in.Severity)
|
|
case strings.TrimSpace(in.Title) == "" || len(in.Title) > maxTitleLen:
|
|
return fmt.Errorf("%w: title is required and at most %d bytes", ErrInvalid, maxTitleLen)
|
|
case len(in.Body) > maxBodyLen:
|
|
return fmt.Errorf("%w: body is at most %d bytes", ErrInvalid, maxBodyLen)
|
|
case len(in.Link) > maxLinkLen || (in.Link != "" && !strings.HasPrefix(in.Link, "/")):
|
|
return fmt.Errorf("%w: link must be a dashboard path of at most %d bytes", ErrInvalid, maxLinkLen)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Alert is one row.
|
|
type Alert struct {
|
|
ID int64
|
|
Key
|
|
Severity Severity
|
|
Title string
|
|
Body string
|
|
Link string
|
|
Condition bool
|
|
State State
|
|
FirstRaised time.Time
|
|
LastSeen time.Time
|
|
Count int64
|
|
// ClosedBy and ClosedAt say who resolved or dismissed the alert and when.
|
|
ClosedBy string
|
|
ClosedAt time.Time
|
|
// Deliveries are the outbound notices this alert owed, newest first.
|
|
Deliveries []Delivery
|
|
}
|
|
|
|
// Apply is the alert after a raise. prev is the existing row, nil when there
|
|
// is none. opened reports that the alert became open: it is new, or it was
|
|
// resolved, or it was an event alert that had been dismissed. An open alert
|
|
// raised again only bumps its count and last seen. A dismissed condition
|
|
// alert is snoozed: it stays dismissed until the condition clears and recurs.
|
|
func Apply(prev *Alert, in Input, now time.Time) (next Alert, opened bool) {
|
|
fresh := Alert{
|
|
Key: in.Key, Severity: in.Severity, Title: in.Title, Body: in.Body, Link: in.Link,
|
|
Condition: in.Condition, State: StateOpen, FirstRaised: now, LastSeen: now, Count: 1,
|
|
}
|
|
if prev == nil {
|
|
return fresh, true
|
|
}
|
|
next = *prev
|
|
next.Severity, next.Title, next.Body, next.Link, next.Condition = in.Severity, in.Title, in.Body, in.Link, in.Condition
|
|
next.LastSeen = now
|
|
switch {
|
|
case prev.State == StateOpen:
|
|
next.Count++
|
|
return next, false
|
|
case prev.State == StateDismissed && prev.Condition && in.Condition:
|
|
return next, false
|
|
default:
|
|
fresh.ID, fresh.Deliveries = prev.ID, prev.Deliveries
|
|
return fresh, true
|
|
}
|
|
}
|
|
|
|
// Resolve is the alert after its condition cleared. changed reports that it
|
|
// moved to resolved. Only a condition alert has a condition to clear: an
|
|
// event alert stays until dismissed, and a resolved alert stays as it is. A
|
|
// snoozed (dismissed) condition alert resolves, so it can come back if the
|
|
// condition recurs.
|
|
func Resolve(prev Alert, by string, now time.Time) (next Alert, changed bool) {
|
|
if !prev.Condition || prev.State == StateResolved {
|
|
return prev, false
|
|
}
|
|
prev.State, prev.ClosedBy, prev.ClosedAt = StateResolved, by, now
|
|
return prev, true
|
|
}
|
|
|
|
// Dismiss is the alert after an operator dismissed it. changed reports that it
|
|
// moved to dismissed; only an open alert can be dismissed.
|
|
func Dismiss(prev Alert, by string, now time.Time) (next Alert, changed bool) {
|
|
if prev.State != StateOpen {
|
|
return prev, false
|
|
}
|
|
prev.State, prev.ClosedBy, prev.ClosedAt = StateDismissed, by, now
|
|
return prev, true
|
|
}
|
|
|
|
// Filter selects what List returns.
|
|
type Filter struct {
|
|
// States to include; empty means open only.
|
|
States []State
|
|
Limit int
|
|
}
|
|
|
|
// Store is the State Store's alert surface.
|
|
type Store interface {
|
|
Raiser
|
|
// Dismiss dismisses an open alert in org.
|
|
DismissAlert(ctx context.Context, orgID org.OrgID, id int64, by string) error
|
|
// List returns an org's alerts, highest severity then newest first.
|
|
ListAlerts(ctx context.Context, orgID org.OrgID, filter Filter) ([]Alert, error)
|
|
Deliverer
|
|
}
|
|
|
|
// Pruner removes alerts that closed before a retention cutoff, with their
|
|
// deliveries. The State Store runs it; it is not part of the contract.
|
|
type Pruner interface {
|
|
PruneAlerts(ctx context.Context, cutoff time.Time) (int64, error)
|
|
}
|
|
|
|
// Raiser is what a service needs to raise and clear alerts.
|
|
type Raiser interface {
|
|
// RaiseAlert stores in, idempotently: see Apply.
|
|
RaiseAlert(ctx context.Context, in Input) (Alert, error)
|
|
// ResolveAlert clears a condition alert; no alert under key is not an
|
|
// error.
|
|
ResolveAlert(ctx context.Context, key Key, by string) error
|
|
}
|
|
|
|
// Reporter raises and clears alerts for a service. A failure is logged and
|
|
// left for the next check: reporting never blocks or fails the caller's work.
|
|
// The zero Reporter, and one with no Raiser, does nothing.
|
|
type Reporter struct {
|
|
Raiser Raiser
|
|
Log *slog.Logger
|
|
// By names the resolver recorded on alerts this reporter clears.
|
|
By string
|
|
}
|
|
|
|
// Raise stores in.
|
|
func (r Reporter) Raise(ctx context.Context, in Input) {
|
|
if r.Raiser == nil {
|
|
return
|
|
}
|
|
if _, err := r.Raiser.RaiseAlert(ctx, in); err != nil {
|
|
r.warn("alert not raised", in.Key, err)
|
|
}
|
|
}
|
|
|
|
// Resolve clears the alert under key.
|
|
func (r Reporter) Resolve(ctx context.Context, key Key) {
|
|
if r.Raiser == nil {
|
|
return
|
|
}
|
|
if err := r.Raiser.ResolveAlert(ctx, key, r.By); err != nil {
|
|
r.warn("alert not resolved", key, err)
|
|
}
|
|
}
|
|
|
|
// Sync raises in while active holds and resolves it once it does not. It is
|
|
// the whole body of a periodic check for one condition.
|
|
func (r Reporter) Sync(ctx context.Context, active bool, in Input) {
|
|
if active {
|
|
r.Raise(ctx, in)
|
|
return
|
|
}
|
|
r.Resolve(ctx, in.Key)
|
|
}
|
|
|
|
func (r Reporter) warn(msg string, key Key, err error) {
|
|
if r.Log != nil {
|
|
r.Log.Warn(msg, "org", key.Org, "kind", key.Kind, "subject", key.Subject, "err", err)
|
|
}
|
|
}
|
|
|
|
// ErrNotFound is returned for an alert or delivery that does not exist, or
|
|
// that belongs to another org.
|
|
var ErrNotFound = errors.New("alert: not found")
|