Files
archie/modules/events/internal/store/store.go
T
samandCopilot 64dfad6d53 feat(runs): add typed results and model aliases (#2043)
Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-11-18 23:47:35 +11:00

365 lines
14 KiB
Go

// Package store holds the events module's SQL: sources, the events table that
// is Archie's one stream, triggers and the record of which run each trigger
// started.
package store
import (
"context"
"database/sql"
"encoding/json/jsontext"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
)
// ErrNotFound means the named row doesn't exist.
var ErrNotFound = errors.New("not found")
// Source is a named place events come from, read by one connector.
type Source struct {
OrgID string `db:"org_id"`
Name string `db:"name"`
Connector string `db:"connector"`
TokenSHA256 string `db:"token_sha256"`
// Cursor is where a listening source resumes.
Cursor string `db:"cursor"`
// KeyFields are the paths its events are grouped by; none means the default.
KeyFields jsontext.Value `db:"key_fields"`
}
// Event is one stored event.
type Event struct {
ID uuid.UUID `db:"id"`
OrgID string `db:"org_id"`
Source string `db:"source"`
Type string `db:"type"`
SourceEventID string `db:"source_event_id"`
LinkKey sql.NullString `db:"link_key"`
LinkedTo *uuid.UUID `db:"linked_to"`
Fields jsontext.Value `db:"fields"`
// Body is the payload as it arrived: the webhook body, or the batch a
// listening connector streamed. It is what an audit reads when the
// parsed fields leave a question open.
Body []byte `db:"body"`
OccurredAt sql.NullTime `db:"occurred_at"`
ReceivedAt time.Time `db:"received_at"`
Matched bool `db:"matched"`
DispatchedAt sql.NullTime `db:"dispatched_at"`
// Own marks the echo of a delivery Archie sent, and OriginRunID is the run
// that sent it. An own event has no group.
Own bool `db:"own"`
OriginRunID uuid.NullUUID `db:"origin_run_id"`
GroupKey sql.NullString `db:"group_key"`
// Scan into []byte to preserve SQL NULL for events with no group.
GroupFilter []byte `db:"group_filter"`
}
// Trigger starts a workflow for events of one type from one source whose
// fields match its filter.
type Trigger struct {
OrgID string `db:"org_id"`
Name string `db:"name"`
Source string `db:"source"`
EventType string `db:"event_type"`
Filter jsontext.Value `db:"filter"`
Workflow string `db:"workflow"`
Enabled bool `db:"enabled"`
// ScheduleEverySeconds is the interval a scheduled trigger fires on; it
// is null for a trigger that fires on events.
ScheduleEverySeconds sql.NullInt64 `db:"schedule_every_s"`
// ScheduleSince is when the schedule began or last changed.
ScheduleSince sql.NullTime `db:"schedule_since"`
}
// Store is the events module's Postgres store.
type Store struct {
db *sqlx.DB
}
// New returns a store on db.
func New(db *sqlx.DB) *Store { return &Store{db: db} }
// Source returns the named source.
func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) {
var src Source
err := s.db.GetContext(ctx, &src, `
SELECT org_id, name, connector, token_sha256, key_fields FROM events.sources
WHERE org_id = $1 AND name = $2`, orgID, name)
if errors.Is(err, sql.ErrNoRows) {
return Source{}, ErrNotFound
}
if err != nil {
return Source{}, fmt.Errorf("read source: %w", err)
}
return src, nil
}
// Sources returns every source of the organisation.
func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) {
var out []Source
if err := s.db.SelectContext(ctx, &out, `
SELECT org_id, name, connector, token_sha256, cursor, key_fields FROM events.sources
WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
return nil, fmt.Errorf("list sources: %w", err)
}
return out, nil
}
// SetCursor records where a listening source resumes.
func (s *Store) SetCursor(ctx context.Context, orgID, name, cursor string) error {
if _, err := s.db.ExecContext(ctx, `
UPDATE events.sources SET cursor = $3, updated_at = now() WHERE org_id = $1 AND name = $2`,
orgID, name, cursor); err != nil {
return fmt.Errorf("save cursor: %w", err)
}
return nil
}
// UpsertSource creates a source or replaces its connector and token.
func (s *Store) UpsertSource(ctx context.Context, src Source) error {
if _, err := s.db.NamedExecContext(ctx, `
INSERT INTO events.sources (org_id, name, connector, token_sha256)
VALUES (:org_id, :name, :connector, :token_sha256)
ON CONFLICT (org_id, name) DO UPDATE
SET connector = EXCLUDED.connector, token_sha256 = EXCLUDED.token_sha256, updated_at = now()`, src); err != nil {
return fmt.Errorf("save source: %w", err)
}
return nil
}
// Insert stores an event unless its source already delivered the same source
// event id. It reports whether the event was new.
func (s *Store) Insert(ctx context.Context, e Event) (bool, error) {
res, err := s.db.NamedExecContext(ctx, `
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, body, occurred_at,
own, origin_run_id, group_key, group_filter)
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :body, :occurred_at,
:own, :origin_run_id, :group_key, :group_filter)
ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e)
if err != nil {
return false, fmt.Errorf("store event: %w", err)
}
n, err := res.RowsAffected()
if err != nil {
return false, fmt.Errorf("store event: %w", err)
}
return n == 1, nil
}
// Dispatch claims the oldest undispatched event and hands it to fn inside one
// transaction. fn's matched result and the dispatched mark commit together,
// so an event is dispatched once even with several workers. It reports
// whether there was an event to dispatch.
func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (matched bool, err error)) (bool, error) {
tx, err := s.db.BeginTxx(ctx, nil)
if err != nil {
return false, fmt.Errorf("begin dispatch: %w", err)
}
defer func() { _ = tx.Rollback() }()
var e Event
err = tx.GetContext(ctx, &e, `
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
FROM events.events WHERE dispatched_at IS NULL
ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`)
if errors.Is(err, sql.ErrNoRows) {
return false, nil
}
if err != nil {
return false, fmt.Errorf("claim event: %w", err)
}
matched, err := fn(tx, e)
if err != nil {
return true, err
}
if _, err := tx.ExecContext(ctx, `
UPDATE events.events SET dispatched_at = now(), matched = $2 WHERE id = $1`, e.ID, matched); err != nil {
return true, fmt.Errorf("mark dispatched: %w", err)
}
if err := tx.Commit(); err != nil {
return true, fmt.Errorf("commit dispatch: %w", err)
}
return true, nil
}
// Link points e at the earliest event its source delivered before it with the
// same link key, and reports whether it found one. Only an earlier event
// counts: a firing alert must not link to the resolve that follows it.
func Link(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) {
if !e.LinkKey.Valid {
return false, nil
}
const earlier = `org_id = $2 AND source = $3 AND link_key = $4 AND id <> $1
AND (received_at, id) < ($5, $1)`
res, err := tx.ExecContext(ctx, `
UPDATE events.events SET linked_to = (
SELECT id FROM events.events WHERE `+earlier+`
ORDER BY received_at, id LIMIT 1)
WHERE id = $1 AND EXISTS (SELECT 1 FROM events.events WHERE `+earlier+`)`,
e.ID, e.OrgID, e.Source, e.LinkKey.String, e.ReceivedAt)
if err != nil {
return false, fmt.Errorf("link event: %w", err)
}
n, err := res.RowsAffected()
return n == 1, err
}
// Triggers returns the enabled triggers for one source and event type.
func Triggers(ctx context.Context, tx *sqlx.Tx, orgID, source, eventType string) ([]Trigger, error) {
var out []Trigger
if err := tx.SelectContext(ctx, &out, `
SELECT org_id, name, source, event_type, filter, workflow, enabled FROM events.triggers
WHERE org_id = $1 AND source = $2 AND event_type = $3 AND enabled ORDER BY name`,
orgID, source, eventType); err != nil {
return nil, fmt.Errorf("read triggers: %w", err)
}
return out, nil
}
// RecordDispatch notes that trigger started run for an event.
func RecordDispatch(ctx context.Context, tx *sqlx.Tx, eventID uuid.UUID, trigger string, runID uuid.UUID) error {
if _, err := tx.ExecContext(ctx, `
INSERT INTO events.dispatches (event_id, trigger, run_id) VALUES ($1, $2, $3)
ON CONFLICT (event_id, trigger) DO NOTHING`, eventID, trigger, runID); err != nil {
return fmt.Errorf("record dispatch: %w", err)
}
return nil
}
// UpsertTrigger creates or replaces a trigger.
func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error {
if _, err := s.db.NamedExecContext(ctx, `
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled,
schedule_every_s, schedule_since)
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled,
CAST(:schedule_every_s AS integer), CASE WHEN CAST(:schedule_every_s AS integer) IS NULL THEN NULL ELSE now() END)
ON CONFLICT (org_id, name) DO UPDATE
SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter,
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled,
schedule_since = CASE
WHEN EXCLUDED.schedule_every_s IS NULL THEN NULL
WHEN events.triggers.schedule_every_s IS NOT DISTINCT FROM EXCLUDED.schedule_every_s
THEN events.triggers.schedule_since
ELSE now() END,
schedule_every_s = EXCLUDED.schedule_every_s, updated_at = now()`, t); err != nil {
return fmt.Errorf("save trigger: %w", err)
}
return nil
}
// ScheduledTriggers returns the enabled triggers that fire on an interval.
func (s *Store) ScheduledTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
var out []Trigger
if err := s.db.SelectContext(ctx, &out, `
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
FROM events.triggers
WHERE org_id = $1 AND enabled AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil {
return nil, fmt.Errorf("read schedules: %w", err)
}
return out, nil
}
// EventFilter narrows the events a list returns. A zero field doesn't filter,
// and Matched distinguishes events a trigger started a run for from the rest.
type EventFilter struct {
Source string
Type string
Matched *bool
Limit int
}
// Recent returns an organisation's newest events that match filter.
func (s *Store) Recent(ctx context.Context, orgID string, f EventFilter) ([]Event, error) {
var out []Event
if err := s.db.SelectContext(ctx, &out, `
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
FROM events.events
WHERE org_id = $1
AND ($2 = '' OR source = $2)
AND ($3 = '' OR type = $3)
AND ($4::boolean IS NULL OR matched = $4)
ORDER BY received_at DESC LIMIT $5`, orgID, f.Source, f.Type, f.Matched, f.Limit); err != nil {
return nil, fmt.Errorf("list events: %w", err)
}
return out, nil
}
// Event returns one event with the payload it arrived in.
func (s *Store) Event(ctx context.Context, orgID string, id uuid.UUID) (Event, error) {
var e Event
err := s.db.GetContext(ctx, &e, `
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, body, occurred_at,
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
FROM events.events WHERE org_id = $1 AND id = $2`, orgID, id)
if errors.Is(err, sql.ErrNoRows) {
return Event{}, ErrNotFound
}
if err != nil {
return Event{}, fmt.Errorf("read event: %w", err)
}
return e, nil
}
// TriggerRun is a run a trigger started for an event.
type TriggerRun struct {
Trigger string `db:"trigger"`
RunID uuid.UUID `db:"run_id"`
}
// RunsForEvent returns the runs triggers started for an event.
func (s *Store) RunsForEvent(ctx context.Context, orgID string, id uuid.UUID) ([]TriggerRun, error) {
var out []TriggerRun
if err := s.db.SelectContext(ctx, &out, `
SELECT d.trigger, d.run_id FROM events.dispatches d
JOIN events.events e ON e.id = d.event_id
WHERE e.org_id = $1 AND d.event_id = $2 ORDER BY d.trigger`, orgID, id); err != nil {
return nil, fmt.Errorf("read event runs: %w", err)
}
return out, nil
}
// AllTriggers returns every trigger of the organisation, enabled or not.
func (s *Store) AllTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
var out []Trigger
if err := s.db.SelectContext(ctx, &out, `
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
FROM events.triggers WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
return nil, fmt.Errorf("list triggers: %w", err)
}
return out, nil
}
// Origin is the event and trigger that started a run.
type Origin struct {
EventID uuid.UUID `db:"event_id"`
Source string `db:"source"`
Type string `db:"type"`
Fields jsontext.Value `db:"fields"`
ReceivedAt time.Time `db:"received_at"`
Trigger string `db:"trigger"`
Filter jsontext.Value `db:"filter"`
}
// OriginOf returns what started a run; ok is false when no trigger did.
func (s *Store) OriginOf(ctx context.Context, orgID string, runID uuid.UUID) (Origin, bool, error) {
var o Origin
err := s.db.GetContext(ctx, &o, `
SELECT e.id AS event_id, e.source, e.type, e.fields, e.received_at, d.trigger,
COALESCE(t.filter, '{}') AS filter
FROM events.dispatches d
JOIN events.events e ON e.id = d.event_id
LEFT JOIN events.triggers t ON t.org_id = e.org_id AND t.name = d.trigger
WHERE e.org_id = $1 AND d.run_id = $2`, orgID, runID)
if errors.Is(err, sql.ErrNoRows) {
return Origin{}, false, nil
}
if err != nil {
return Origin{}, false, fmt.Errorf("read run origin: %w", err)
}
return o, true, nil
}