332 lines
10 KiB
Go
332 lines
10 KiB
Go
// Operator alerts: the Postgres implementation of internal/domain/alert.Store.
|
|
// Every write runs in a transaction that holds an advisory lock on the alert's
|
|
// key, so a raise and a resolve of one alert never interleave, and records the
|
|
// event and the delivery the change owes in the same commit.
|
|
package postgres
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/alert"
|
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
|
|
)
|
|
|
|
var (
|
|
_ alert.Store = (*Store)(nil)
|
|
_ alert.Pruner = (*Store)(nil)
|
|
)
|
|
|
|
// deliveriesPerAlert bounds the delivery history one listed alert carries.
|
|
const deliveriesPerAlert = 10
|
|
|
|
func alertFromRow(r postgresdb.Alert) alert.Alert {
|
|
a := alert.Alert{
|
|
ID: r.ID,
|
|
Org: org.OrgID(r.OrgID), Kind: r.Kind, Subject: r.Subject,
|
|
Severity: alert.Severity(r.Severity), Title: r.Title, Body: r.Body, Link: r.Link,
|
|
Condition: r.IsCondition, State: alert.State(r.State),
|
|
FirstRaised: r.FirstRaised, LastSeen: r.LastSeen, Count: r.Count, ClosedBy: r.ClosedBy,
|
|
}
|
|
if r.ClosedAt.Valid {
|
|
a.ClosedAt = r.ClosedAt.Time
|
|
}
|
|
return a
|
|
}
|
|
|
|
func alertParams(a alert.Alert) postgresdb.UpsertAlertParams {
|
|
return postgresdb.UpsertAlertParams{
|
|
OrgID: string(a.Org), Kind: a.Kind, Subject: a.Subject, Severity: string(a.Severity),
|
|
Title: a.Title, Body: a.Body, Link: a.Link, IsCondition: a.Condition, State: string(a.State),
|
|
FirstRaised: a.FirstRaised, LastSeen: a.LastSeen, Count: a.Count, ClosedBy: a.ClosedBy,
|
|
ClosedAt: pgtype.Timestamptz{Time: a.ClosedAt, Valid: !a.ClosedAt.IsZero()},
|
|
}
|
|
}
|
|
|
|
// Quoted so a NUL in a field is escaped: Postgres text rejects a raw NUL.
|
|
func alertLockKey(k alert.Key) string {
|
|
return fmt.Sprintf("%q|%q|%q", string(k.Org), k.Kind, k.Subject)
|
|
}
|
|
|
|
// lockedAlert takes the key's lock and returns the stored alert, nil when
|
|
// there is none.
|
|
func lockedAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key) (*alert.Alert, error) {
|
|
if err := q.LockAlertKey(ctx, alertLockKey(key)); err != nil {
|
|
return nil, err
|
|
}
|
|
row, err := q.GetAlertByKey(ctx, postgresdb.GetAlertByKeyParams{OrgID: string(key.Org), Kind: key.Kind, Subject: key.Subject})
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
a := alertFromRow(row)
|
|
return &a, nil
|
|
}
|
|
|
|
// alertChanged records an alert's change on the event stream, and the
|
|
// delivery it owes when delivery is not empty.
|
|
func alertChanged(ctx context.Context, q *postgresdb.Queries, a alert.Alert, delivery string) error {
|
|
if delivery != "" {
|
|
if err := q.InsertAlertDelivery(ctx, postgresdb.InsertAlertDeliveryParams{AlertID: a.ID, Event: delivery}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err := insertEventQ(org.WithOrg(ctx, a.Org), q, events.Event{
|
|
Kind: events.KindAlert, Detail: a.Title,
|
|
Data: map[string]any{"id": a.ID, "kind": a.Kind, "state": string(a.State), "severity": string(a.Severity)},
|
|
})
|
|
return err
|
|
}
|
|
|
|
// raiseAlert stores in under q, which must be a transaction.
|
|
func raiseAlert(ctx context.Context, q *postgresdb.Queries, in alert.Input) (alert.Alert, error) {
|
|
if err := in.Validate(); err != nil {
|
|
return alert.Alert{}, err
|
|
}
|
|
prev, err := lockedAlert(ctx, q, in.Key)
|
|
if err != nil {
|
|
return alert.Alert{}, err
|
|
}
|
|
next, opened := alert.Apply(prev, in, time.Now().UTC())
|
|
row, err := q.UpsertAlert(ctx, alertParams(next))
|
|
if err != nil {
|
|
return alert.Alert{}, err
|
|
}
|
|
stored := alertFromRow(row)
|
|
if opened {
|
|
if err := alertChanged(ctx, q, stored, alert.EventRaised); err != nil {
|
|
return alert.Alert{}, err
|
|
}
|
|
}
|
|
return stored, nil
|
|
}
|
|
|
|
// resolveAlert clears the alert under key through q, which must be a
|
|
// transaction.
|
|
func resolveAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key, by string) error {
|
|
prev, err := lockedAlert(ctx, q, key)
|
|
if err != nil || prev == nil {
|
|
return err
|
|
}
|
|
next, changed := alert.Resolve(*prev, by, time.Now().UTC())
|
|
if !changed {
|
|
return nil
|
|
}
|
|
row, err := q.UpsertAlert(ctx, alertParams(next))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return alertChanged(ctx, q, alertFromRow(row), alert.EventResolved)
|
|
}
|
|
|
|
func (s *Store) inTx(ctx context.Context, fn func(*postgresdb.Queries) error) error {
|
|
tx, err := s.pool.Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback(ctx) }()
|
|
if err := fn(postgresdb.New(tx)); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
// Raise stores in idempotently.
|
|
func (s *Store) RaiseAlert(ctx context.Context, in alert.Input) (alert.Alert, error) {
|
|
var out alert.Alert
|
|
err := s.inTx(ctx, func(q *postgresdb.Queries) (err error) {
|
|
out, err = raiseAlert(ctx, q, in)
|
|
return err
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
// Resolve clears the alert under key.
|
|
func (s *Store) ResolveAlert(ctx context.Context, key alert.Key, by string) error {
|
|
return s.inTx(ctx, func(q *postgresdb.Queries) error { return resolveAlert(ctx, q, key, by) })
|
|
}
|
|
|
|
// Dismiss dismisses an open alert of orgID. An alert of another org is
|
|
// indistinguishable from none.
|
|
func (s *Store) DismissAlert(ctx context.Context, orgID org.OrgID, id int64, by string) error {
|
|
return s.inTx(ctx, func(q *postgresdb.Queries) error {
|
|
row, err := q.GetOrgAlert(ctx, postgresdb.GetOrgAlertParams{OrgID: string(orgID), ID: id})
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return alert.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
prev, err := lockedAlert(ctx, q, alertFromRow(row).Key)
|
|
if err != nil || prev == nil {
|
|
return errors.Join(err, alert.ErrNotFound)
|
|
}
|
|
next, changed := alert.Dismiss(*prev, by, time.Now().UTC())
|
|
if !changed {
|
|
return nil
|
|
}
|
|
stored, err := q.UpsertAlert(ctx, alertParams(next))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return alertChanged(ctx, q, alertFromRow(stored), "")
|
|
})
|
|
}
|
|
|
|
// List returns an org's alerts with their deliveries.
|
|
func (s *Store) ListAlerts(ctx context.Context, orgID org.OrgID, filter alert.Filter) ([]alert.Alert, error) {
|
|
states := []string{string(alert.StateOpen)}
|
|
if len(filter.States) > 0 {
|
|
states = states[:0]
|
|
for _, st := range filter.States {
|
|
states = append(states, string(st))
|
|
}
|
|
}
|
|
limit := filter.Limit
|
|
if limit <= 0 || limit > 500 {
|
|
limit = 200
|
|
}
|
|
q := s.queries()
|
|
rows, err := q.ListAlerts(ctx, postgresdb.ListAlertsParams{OrgID: string(orgID), States: states, PageLimit: int32(limit)})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]alert.Alert, len(rows))
|
|
ids := make([]int64, len(rows))
|
|
for i, r := range rows {
|
|
out[i], ids[i] = alertFromRow(r), r.ID
|
|
}
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
deliveries, err := alertDeliveries(ctx, q, ids)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range out {
|
|
out[i].Deliveries = deliveries[out[i].ID]
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// alertDeliveries returns each alert's newest deliveries with their attempts.
|
|
func alertDeliveries(ctx context.Context, q *postgresdb.Queries, alertIDs []int64) (map[int64][]alert.Delivery, error) {
|
|
rows, err := q.ListAlertDeliveries(ctx, alertIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
byAlert := map[int64][]alert.Delivery{}
|
|
var deliveryIDs []int64
|
|
for _, r := range rows {
|
|
if len(byAlert[r.AlertID]) >= deliveriesPerAlert {
|
|
continue
|
|
}
|
|
byAlert[r.AlertID] = append(byAlert[r.AlertID], deliveryFromRow(r))
|
|
deliveryIDs = append(deliveryIDs, r.ID)
|
|
}
|
|
if len(deliveryIDs) == 0 {
|
|
return byAlert, nil
|
|
}
|
|
attempts, err := q.ListAlertAttempts(ctx, deliveryIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
byDelivery := map[int64][]alert.Attempt{}
|
|
for _, a := range attempts {
|
|
byDelivery[a.DeliveryID] = append(byDelivery[a.DeliveryID], alert.Attempt{
|
|
N: a.Attempt, At: a.At, StatusCode: int(a.StatusCode), Error: a.Error,
|
|
})
|
|
}
|
|
for _, list := range byAlert {
|
|
for i := range list {
|
|
list[i].Attempts = byDelivery[list[i].ID]
|
|
}
|
|
}
|
|
return byAlert, nil
|
|
}
|
|
|
|
func deliveryFromRow(r postgresdb.AlertDelivery) alert.Delivery {
|
|
return alert.Delivery{
|
|
ID: r.ID, AlertID: r.AlertID, Event: r.Event, State: r.State,
|
|
CreatedAt: r.CreatedAt, NextAttemptAt: r.NextAttemptAt,
|
|
}
|
|
}
|
|
|
|
// Prune removes alerts closed before cutoff, with their deliveries.
|
|
func (s *Store) PruneAlerts(ctx context.Context, cutoff time.Time) (int64, error) {
|
|
return s.queries().PruneAlerts(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true})
|
|
}
|
|
|
|
// ClaimDeliveries returns the due deliveries with their alerts.
|
|
func (s *Store) ClaimAlertDeliveries(ctx context.Context, limit int) ([]alert.Due, error) {
|
|
if limit <= 0 || limit > 100 {
|
|
limit = 20
|
|
}
|
|
var out []alert.Due
|
|
err := s.inTx(ctx, func(q *postgresdb.Queries) error {
|
|
rows, err := q.ClaimAlertDeliveries(ctx, int32(limit))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, r := range rows {
|
|
row, err := q.GetAlert(ctx, r.AlertID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out = append(out, alert.Due{Delivery: deliveryFromRow(r), Alert: alertFromRow(row), Attempt: r.Attempts + 1})
|
|
}
|
|
return nil
|
|
})
|
|
return out, err
|
|
}
|
|
|
|
// RecordAttempt records one attempt and settles or reschedules the delivery.
|
|
func (s *Store) RecordAlertAttempt(ctx context.Context, o alert.Outcome) error {
|
|
return s.inTx(ctx, func(q *postgresdb.Queries) error {
|
|
d, err := q.LockAlertDelivery(ctx, o.DeliveryID)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return alert.ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.State != alert.DeliveryPending {
|
|
return nil
|
|
}
|
|
now := time.Now().UTC()
|
|
params := postgresdb.SettleAlertDeliveryParams{ID: d.ID, State: alert.DeliveryPending, Attempts: d.Attempts, NextAttemptAt: d.NextAttemptAt}
|
|
switch {
|
|
case o.Skipped:
|
|
params.State = alert.DeliverySkipped
|
|
default:
|
|
params.Attempts++
|
|
if err := q.InsertAlertAttempt(ctx, postgresdb.InsertAlertAttemptParams{
|
|
DeliveryID: d.ID, Attempt: params.Attempts, StatusCode: int64(o.StatusCode), Error: o.Error,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
switch {
|
|
case o.OK():
|
|
params.State = alert.DeliveryDelivered
|
|
case params.Attempts >= alert.MaxDeliveryAttempts:
|
|
params.State = alert.DeliveryFailed
|
|
default:
|
|
params.NextAttemptAt = now.Add(alert.RetryDelay(params.Attempts))
|
|
}
|
|
}
|
|
if err := q.SettleAlertDelivery(ctx, params); err != nil {
|
|
return fmt.Errorf("settle alert delivery: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
}
|