// Operator alerts: the Postgres implementation of internal/domain/alert.Store. // Every write runs in a transaction that holds an advisory lock on the alert's // key, so a raise and a resolve of one alert never interleave, and records the // event and the delivery the change owes in the same commit. package postgres import ( "context" "errors" "fmt" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/samcharles93/archie-core/internal/domain/alert" "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" ) var ( _ alert.Store = (*Store)(nil) _ alert.Pruner = (*Store)(nil) ) // deliveriesPerAlert bounds the delivery history one listed alert carries. const deliveriesPerAlert = 10 func alertFromRow(r postgresdb.Alert) alert.Alert { a := alert.Alert{ ID: r.ID, Org: org.OrgID(r.OrgID), Kind: r.Kind, Subject: r.Subject, Severity: alert.Severity(r.Severity), Title: r.Title, Body: r.Body, Link: r.Link, Condition: r.IsCondition, State: alert.State(r.State), FirstRaised: r.FirstRaised, LastSeen: r.LastSeen, Count: r.Count, ClosedBy: r.ClosedBy, } if r.ClosedAt.Valid { a.ClosedAt = r.ClosedAt.Time } return a } func alertParams(a alert.Alert) postgresdb.UpsertAlertParams { return postgresdb.UpsertAlertParams{ OrgID: string(a.Org), Kind: a.Kind, Subject: a.Subject, Severity: string(a.Severity), Title: a.Title, Body: a.Body, Link: a.Link, IsCondition: a.Condition, State: string(a.State), FirstRaised: a.FirstRaised, LastSeen: a.LastSeen, Count: a.Count, ClosedBy: a.ClosedBy, ClosedAt: pgtype.Timestamptz{Time: a.ClosedAt, Valid: !a.ClosedAt.IsZero()}, } } // Quoted so a NUL in a field is escaped: Postgres text rejects a raw NUL. func alertLockKey(k alert.Key) string { return fmt.Sprintf("%q|%q|%q", string(k.Org), k.Kind, k.Subject) } // lockedAlert takes the key's lock and returns the stored alert, nil when // there is none. func lockedAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key) (*alert.Alert, error) { if err := q.LockAlertKey(ctx, alertLockKey(key)); err != nil { return nil, err } row, err := q.GetAlertByKey(ctx, postgresdb.GetAlertByKeyParams{OrgID: string(key.Org), Kind: key.Kind, Subject: key.Subject}) if errors.Is(err, pgx.ErrNoRows) { return nil, nil } if err != nil { return nil, err } a := alertFromRow(row) return &a, nil } // alertChanged records an alert's change on the event stream, and the // delivery it owes when delivery is not empty. func alertChanged(ctx context.Context, q *postgresdb.Queries, a alert.Alert, delivery string) error { if delivery != "" { if err := q.InsertAlertDelivery(ctx, postgresdb.InsertAlertDeliveryParams{AlertID: a.ID, Event: delivery}); err != nil { return err } } _, err := insertEventQ(org.WithOrg(ctx, a.Org), q, events.Event{ Kind: events.KindAlert, Detail: a.Title, Data: map[string]any{"id": a.ID, "kind": a.Kind, "state": string(a.State), "severity": string(a.Severity)}, }) return err } // raiseAlert stores in under q, which must be a transaction. func raiseAlert(ctx context.Context, q *postgresdb.Queries, in alert.Input) (alert.Alert, error) { if err := in.Validate(); err != nil { return alert.Alert{}, err } prev, err := lockedAlert(ctx, q, in.Key) if err != nil { return alert.Alert{}, err } next, opened := alert.Apply(prev, in, time.Now().UTC()) row, err := q.UpsertAlert(ctx, alertParams(next)) if err != nil { return alert.Alert{}, err } stored := alertFromRow(row) if opened { if err := alertChanged(ctx, q, stored, alert.EventRaised); err != nil { return alert.Alert{}, err } } return stored, nil } // resolveAlert clears the alert under key through q, which must be a // transaction. func resolveAlert(ctx context.Context, q *postgresdb.Queries, key alert.Key, by string) error { prev, err := lockedAlert(ctx, q, key) if err != nil || prev == nil { return err } next, changed := alert.Resolve(*prev, by, time.Now().UTC()) if !changed { return nil } row, err := q.UpsertAlert(ctx, alertParams(next)) if err != nil { return err } return alertChanged(ctx, q, alertFromRow(row), alert.EventResolved) } func (s *Store) inTx(ctx context.Context, fn func(*postgresdb.Queries) error) error { tx, err := s.pool.Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() if err := fn(postgresdb.New(tx)); err != nil { return err } return tx.Commit(ctx) } // Raise stores in idempotently. func (s *Store) RaiseAlert(ctx context.Context, in alert.Input) (alert.Alert, error) { var out alert.Alert err := s.inTx(ctx, func(q *postgresdb.Queries) (err error) { out, err = raiseAlert(ctx, q, in) return err }) return out, err } // Resolve clears the alert under key. func (s *Store) ResolveAlert(ctx context.Context, key alert.Key, by string) error { return s.inTx(ctx, func(q *postgresdb.Queries) error { return resolveAlert(ctx, q, key, by) }) } // Dismiss dismisses an open alert of orgID. An alert of another org is // indistinguishable from none. func (s *Store) DismissAlert(ctx context.Context, orgID org.OrgID, id int64, by string) error { return s.inTx(ctx, func(q *postgresdb.Queries) error { row, err := q.GetOrgAlert(ctx, postgresdb.GetOrgAlertParams{OrgID: string(orgID), ID: id}) if errors.Is(err, pgx.ErrNoRows) { return alert.ErrNotFound } if err != nil { return err } prev, err := lockedAlert(ctx, q, alertFromRow(row).Key) if err != nil || prev == nil { return errors.Join(err, alert.ErrNotFound) } next, changed := alert.Dismiss(*prev, by, time.Now().UTC()) if !changed { return nil } stored, err := q.UpsertAlert(ctx, alertParams(next)) if err != nil { return err } return alertChanged(ctx, q, alertFromRow(stored), "") }) } // List returns an org's alerts with their deliveries. func (s *Store) ListAlerts(ctx context.Context, orgID org.OrgID, filter alert.Filter) ([]alert.Alert, error) { states := []string{string(alert.StateOpen)} if len(filter.States) > 0 { states = states[:0] for _, st := range filter.States { states = append(states, string(st)) } } limit := filter.Limit if limit <= 0 || limit > 500 { limit = 200 } q := s.queries() rows, err := q.ListAlerts(ctx, postgresdb.ListAlertsParams{OrgID: string(orgID), States: states, PageLimit: int32(limit)}) if err != nil { return nil, err } out := make([]alert.Alert, len(rows)) ids := make([]int64, len(rows)) for i, r := range rows { out[i], ids[i] = alertFromRow(r), r.ID } if len(ids) == 0 { return nil, nil } deliveries, err := alertDeliveries(ctx, q, ids) if err != nil { return nil, err } for i := range out { out[i].Deliveries = deliveries[out[i].ID] } return out, nil } // alertDeliveries returns each alert's newest deliveries with their attempts. func alertDeliveries(ctx context.Context, q *postgresdb.Queries, alertIDs []int64) (map[int64][]alert.Delivery, error) { rows, err := q.ListAlertDeliveries(ctx, alertIDs) if err != nil { return nil, err } byAlert := map[int64][]alert.Delivery{} var deliveryIDs []int64 for _, r := range rows { if len(byAlert[r.AlertID]) >= deliveriesPerAlert { continue } byAlert[r.AlertID] = append(byAlert[r.AlertID], deliveryFromRow(r)) deliveryIDs = append(deliveryIDs, r.ID) } if len(deliveryIDs) == 0 { return byAlert, nil } attempts, err := q.ListAlertAttempts(ctx, deliveryIDs) if err != nil { return nil, err } byDelivery := map[int64][]alert.Attempt{} for _, a := range attempts { byDelivery[a.DeliveryID] = append(byDelivery[a.DeliveryID], alert.Attempt{ N: a.Attempt, At: a.At, StatusCode: int(a.StatusCode), Error: a.Error, }) } for _, list := range byAlert { for i := range list { list[i].Attempts = byDelivery[list[i].ID] } } return byAlert, nil } func deliveryFromRow(r postgresdb.AlertDelivery) alert.Delivery { return alert.Delivery{ ID: r.ID, AlertID: r.AlertID, Event: r.Event, State: r.State, CreatedAt: r.CreatedAt, NextAttemptAt: r.NextAttemptAt, } } // Prune removes alerts closed before cutoff, with their deliveries. func (s *Store) PruneAlerts(ctx context.Context, cutoff time.Time) (int64, error) { return s.queries().PruneAlerts(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } // ClaimDeliveries returns the due deliveries with their alerts. func (s *Store) ClaimAlertDeliveries(ctx context.Context, limit int) ([]alert.Due, error) { if limit <= 0 || limit > 100 { limit = 20 } var out []alert.Due err := s.inTx(ctx, func(q *postgresdb.Queries) error { rows, err := q.ClaimAlertDeliveries(ctx, int32(limit)) if err != nil { return err } for _, r := range rows { row, err := q.GetAlert(ctx, r.AlertID) if err != nil { return err } out = append(out, alert.Due{Delivery: deliveryFromRow(r), Alert: alertFromRow(row), Attempt: r.Attempts + 1}) } return nil }) return out, err } // RecordAttempt records one attempt and settles or reschedules the delivery. func (s *Store) RecordAlertAttempt(ctx context.Context, o alert.Outcome) error { return s.inTx(ctx, func(q *postgresdb.Queries) error { d, err := q.LockAlertDelivery(ctx, o.DeliveryID) if errors.Is(err, pgx.ErrNoRows) { return alert.ErrNotFound } if err != nil { return err } if d.State != alert.DeliveryPending { return nil } now := time.Now().UTC() params := postgresdb.SettleAlertDeliveryParams{ID: d.ID, State: alert.DeliveryPending, Attempts: d.Attempts, NextAttemptAt: d.NextAttemptAt} switch { case o.Skipped: params.State = alert.DeliverySkipped default: params.Attempts++ if err := q.InsertAlertAttempt(ctx, postgresdb.InsertAlertAttemptParams{ DeliveryID: d.ID, Attempt: params.Attempts, StatusCode: int64(o.StatusCode), Error: o.Error, }); err != nil { return err } switch { case o.OK(): params.State = alert.DeliveryDelivered case params.Attempts >= alert.MaxDeliveryAttempts: params.State = alert.DeliveryFailed default: params.NextAttemptAt = now.Add(alert.RetryDelay(params.Attempts)) } } if err := q.SettleAlertDelivery(ctx, params); err != nil { return fmt.Errorf("settle alert delivery: %w", err) } return nil }) }