Files

332 lines
10 KiB
Go

// Operator alerts: the Postgres implementation of internal/domain/alert.Store.
// Every write runs in a transaction that holds an advisory lock on the alert's
// key, so a raise and a resolve of one alert never interleave, and records the
// event and the delivery the change owes in the same commit.
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb"
)
var (
_ alert.Store = (*Store)(nil)
_ alert.Pruner = (*Store)(nil)
)
// deliveriesPerAlert bounds the delivery history one listed alert carries.
const deliveriesPerAlert = 10
func alertFromRow(r postgresdb.Alert) alert.Alert {
a := alert.Alert{
ID: r.ID,
Org: org.OrgID(r.OrgID), Kind: r.Kind, Subject: r.Subject,
Severity: alert.Severity(r.Severity), Title: r.Title, Body: r.Body, Link: r.Link,
Condition: r.IsCondition, State: alert.State(r.State),
FirstRaised: r.FirstRaised, LastSeen: r.LastSeen, Count: r.Count, ClosedBy: r.ClosedBy,
}
if r.ClosedAt.Valid {
a.ClosedAt = r.ClosedAt.Time
}
return a
}
func alertParams(a alert.Alert) postgresdb.UpsertAlertParams {
return postgresdb.UpsertAlertParams{
OrgID: string(a.Org), Kind: a.Kind, Subject: a.Subject, Severity: string(a.Severity),
Title: a.Title, Body: a.Body, Link: a.Link, IsCondition: a.Condition, State: string(a.State),
FirstRaised: a.FirstRaised, LastSeen: a.LastSeen, Count: a.Count, ClosedBy: a.ClosedBy,
ClosedAt: pgtype.Timestamptz{Time: a.ClosedAt, Valid: !a.ClosedAt.IsZero()},
}
}
// Quoted so a NUL in a field is escaped: Postgres text rejects a raw NUL.
func alertLockKey(k alert.Key) string {
return fmt.Sprintf("%q|%q|%q", string(k.Org), k.Kind, k.Subject)
}
// lockedAlert takes the key's lock and returns the stored alert, nil when
// there is none.
func lockedAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key) (*alert.Alert, error) {
if err := q.LockAlertKey(ctx, alertLockKey(key)); err != nil {
return nil, err
}
row, err := q.GetAlertByKey(ctx, postgresdb.GetAlertByKeyParams{OrgID: string(key.Org), Kind: key.Kind, Subject: key.Subject})
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
a := alertFromRow(row)
return &a, nil
}
// alertChanged records an alert's change on the event stream, and the
// delivery it owes when delivery is not empty.
func alertChanged(ctx context.Context, q *postgresdb.Queries, a alert.Alert, delivery string) error {
if delivery != "" {
if err := q.InsertAlertDelivery(ctx, postgresdb.InsertAlertDeliveryParams{AlertID: a.ID, Event: delivery}); err != nil {
return err
}
}
_, err := insertEventQ(org.WithOrg(ctx, a.Org), q, events.Event{
Kind: events.KindAlert, Detail: a.Title,
Data: map[string]any{"id": a.ID, "kind": a.Kind, "state": string(a.State), "severity": string(a.Severity)},
})
return err
}
// raiseAlert stores in under q, which must be a transaction.
func raiseAlert(ctx context.Context, q *postgresdb.Queries, in alert.Input) (alert.Alert, error) {
if err := in.Validate(); err != nil {
return alert.Alert{}, err
}
prev, err := lockedAlert(ctx, q, in.Key)
if err != nil {
return alert.Alert{}, err
}
next, opened := alert.Apply(prev, in, time.Now().UTC())
row, err := q.UpsertAlert(ctx, alertParams(next))
if err != nil {
return alert.Alert{}, err
}
stored := alertFromRow(row)
if opened {
if err := alertChanged(ctx, q, stored, alert.EventRaised); err != nil {
return alert.Alert{}, err
}
}
return stored, nil
}
// resolveAlert clears the alert under key through q, which must be a
// transaction.
func resolveAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key, by string) error {
prev, err := lockedAlert(ctx, q, key)
if err != nil || prev == nil {
return err
}
next, changed := alert.Resolve(*prev, by, time.Now().UTC())
if !changed {
return nil
}
row, err := q.UpsertAlert(ctx, alertParams(next))
if err != nil {
return err
}
return alertChanged(ctx, q, alertFromRow(row), alert.EventResolved)
}
func (s *Store) inTx(ctx context.Context, fn func(*postgresdb.Queries) error) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if err := fn(postgresdb.New(tx)); err != nil {
return err
}
return tx.Commit(ctx)
}
// Raise stores in idempotently.
func (s *Store) RaiseAlert(ctx context.Context, in alert.Input) (alert.Alert, error) {
var out alert.Alert
err := s.inTx(ctx, func(q *postgresdb.Queries) (err error) {
out, err = raiseAlert(ctx, q, in)
return err
})
return out, err
}
// Resolve clears the alert under key.
func (s *Store) ResolveAlert(ctx context.Context, key alert.Key, by string) error {
return s.inTx(ctx, func(q *postgresdb.Queries) error { return resolveAlert(ctx, q, key, by) })
}
// Dismiss dismisses an open alert of orgID. An alert of another org is
// indistinguishable from none.
func (s *Store) DismissAlert(ctx context.Context, orgID org.OrgID, id int64, by string) error {
return s.inTx(ctx, func(q *postgresdb.Queries) error {
row, err := q.GetOrgAlert(ctx, postgresdb.GetOrgAlertParams{OrgID: string(orgID), ID: id})
if errors.Is(err, pgx.ErrNoRows) {
return alert.ErrNotFound
}
if err != nil {
return err
}
prev, err := lockedAlert(ctx, q, alertFromRow(row).Key)
if err != nil || prev == nil {
return errors.Join(err, alert.ErrNotFound)
}
next, changed := alert.Dismiss(*prev, by, time.Now().UTC())
if !changed {
return nil
}
stored, err := q.UpsertAlert(ctx, alertParams(next))
if err != nil {
return err
}
return alertChanged(ctx, q, alertFromRow(stored), "")
})
}
// List returns an org's alerts with their deliveries.
func (s *Store) ListAlerts(ctx context.Context, orgID org.OrgID, filter alert.Filter) ([]alert.Alert, error) {
states := []string{string(alert.StateOpen)}
if len(filter.States) > 0 {
states = states[:0]
for _, st := range filter.States {
states = append(states, string(st))
}
}
limit := filter.Limit
if limit <= 0 || limit > 500 {
limit = 200
}
q := s.queries()
rows, err := q.ListAlerts(ctx, postgresdb.ListAlertsParams{OrgID: string(orgID), States: states, PageLimit: int32(limit)})
if err != nil {
return nil, err
}
out := make([]alert.Alert, len(rows))
ids := make([]int64, len(rows))
for i, r := range rows {
out[i], ids[i] = alertFromRow(r), r.ID
}
if len(ids) == 0 {
return nil, nil
}
deliveries, err := alertDeliveries(ctx, q, ids)
if err != nil {
return nil, err
}
for i := range out {
out[i].Deliveries = deliveries[out[i].ID]
}
return out, nil
}
// alertDeliveries returns each alert's newest deliveries with their attempts.
func alertDeliveries(ctx context.Context, q *postgresdb.Queries, alertIDs []int64) (map[int64][]alert.Delivery, error) {
rows, err := q.ListAlertDeliveries(ctx, alertIDs)
if err != nil {
return nil, err
}
byAlert := map[int64][]alert.Delivery{}
var deliveryIDs []int64
for _, r := range rows {
if len(byAlert[r.AlertID]) >= deliveriesPerAlert {
continue
}
byAlert[r.AlertID] = append(byAlert[r.AlertID], deliveryFromRow(r))
deliveryIDs = append(deliveryIDs, r.ID)
}
if len(deliveryIDs) == 0 {
return byAlert, nil
}
attempts, err := q.ListAlertAttempts(ctx, deliveryIDs)
if err != nil {
return nil, err
}
byDelivery := map[int64][]alert.Attempt{}
for _, a := range attempts {
byDelivery[a.DeliveryID] = append(byDelivery[a.DeliveryID], alert.Attempt{
N: a.Attempt, At: a.At, StatusCode: int(a.StatusCode), Error: a.Error,
})
}
for _, list := range byAlert {
for i := range list {
list[i].Attempts = byDelivery[list[i].ID]
}
}
return byAlert, nil
}
func deliveryFromRow(r postgresdb.AlertDelivery) alert.Delivery {
return alert.Delivery{
ID: r.ID, AlertID: r.AlertID, Event: r.Event, State: r.State,
CreatedAt: r.CreatedAt, NextAttemptAt: r.NextAttemptAt,
}
}
// Prune removes alerts closed before cutoff, with their deliveries.
func (s *Store) PruneAlerts(ctx context.Context, cutoff time.Time) (int64, error) {
return s.queries().PruneAlerts(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true})
}
// ClaimDeliveries returns the due deliveries with their alerts.
func (s *Store) ClaimAlertDeliveries(ctx context.Context, limit int) ([]alert.Due, error) {
if limit <= 0 || limit > 100 {
limit = 20
}
var out []alert.Due
err := s.inTx(ctx, func(q *postgresdb.Queries) error {
rows, err := q.ClaimAlertDeliveries(ctx, int32(limit))
if err != nil {
return err
}
for _, r := range rows {
row, err := q.GetAlert(ctx, r.AlertID)
if err != nil {
return err
}
out = append(out, alert.Due{Delivery: deliveryFromRow(r), Alert: alertFromRow(row), Attempt: r.Attempts + 1})
}
return nil
})
return out, err
}
// RecordAttempt records one attempt and settles or reschedules the delivery.
func (s *Store) RecordAlertAttempt(ctx context.Context, o alert.Outcome) error {
return s.inTx(ctx, func(q *postgresdb.Queries) error {
d, err := q.LockAlertDelivery(ctx, o.DeliveryID)
if errors.Is(err, pgx.ErrNoRows) {
return alert.ErrNotFound
}
if err != nil {
return err
}
if d.State != alert.DeliveryPending {
return nil
}
now := time.Now().UTC()
params := postgresdb.SettleAlertDeliveryParams{ID: d.ID, State: alert.DeliveryPending, Attempts: d.Attempts, NextAttemptAt: d.NextAttemptAt}
switch {
case o.Skipped:
params.State = alert.DeliverySkipped
default:
params.Attempts++
if err := q.InsertAlertAttempt(ctx, postgresdb.InsertAlertAttemptParams{
DeliveryID: d.ID, Attempt: params.Attempts, StatusCode: int64(o.StatusCode), Error: o.Error,
}); err != nil {
return err
}
switch {
case o.OK():
params.State = alert.DeliveryDelivered
case params.Attempts >= alert.MaxDeliveryAttempts:
params.State = alert.DeliveryFailed
default:
params.NextAttemptAt = now.Add(alert.RetryDelay(params.Attempts))
}
}
if err := q.SettleAlertDelivery(ctx, params); err != nil {
return fmt.Errorf("settle alert delivery: %w", err)
}
return nil
})
}