Also fix repository-free message.send validation (#2046) and add a single-binary development smoke path (#1977). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
365 lines
14 KiB
Go
365 lines
14 KiB
Go
// Package store holds the events module's SQL: sources, the events table that
|
|
// is Archie's one stream, triggers and the record of which run each trigger
|
|
// started.
|
|
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json/jsontext"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
)
|
|
|
|
// ErrNotFound means the named row doesn't exist.
|
|
var ErrNotFound = errors.New("not found")
|
|
|
|
// Source is a named place events come from, read by one connector.
|
|
type Source struct {
|
|
OrgID string `db:"org_id"`
|
|
Name string `db:"name"`
|
|
Connector string `db:"connector"`
|
|
TokenSHA256 string `db:"token_sha256"`
|
|
// Cursor is where a listening source resumes.
|
|
Cursor string `db:"cursor"`
|
|
// KeyFields are the paths its events are grouped by; none means the default.
|
|
KeyFields jsontext.Value `db:"key_fields"`
|
|
}
|
|
|
|
// Event is one stored event.
|
|
type Event struct {
|
|
ID uuid.UUID `db:"id"`
|
|
OrgID string `db:"org_id"`
|
|
Source string `db:"source"`
|
|
Type string `db:"type"`
|
|
SourceEventID string `db:"source_event_id"`
|
|
LinkKey sql.NullString `db:"link_key"`
|
|
LinkedTo *uuid.UUID `db:"linked_to"`
|
|
Fields jsontext.Value `db:"fields"`
|
|
// Body is the payload as it arrived: the webhook body, or the batch a
|
|
// listening connector streamed. It is what an audit reads when the
|
|
// parsed fields leave a question open.
|
|
Body []byte `db:"body"`
|
|
OccurredAt sql.NullTime `db:"occurred_at"`
|
|
ReceivedAt time.Time `db:"received_at"`
|
|
Matched bool `db:"matched"`
|
|
DispatchedAt sql.NullTime `db:"dispatched_at"`
|
|
// Own marks the echo of a delivery Archie sent, and OriginRunID is the run
|
|
// that sent it. An own event has no group.
|
|
Own bool `db:"own"`
|
|
OriginRunID uuid.NullUUID `db:"origin_run_id"`
|
|
GroupKey sql.NullString `db:"group_key"`
|
|
// Scan into []byte to preserve SQL NULL for events with no group.
|
|
GroupFilter []byte `db:"group_filter"`
|
|
}
|
|
|
|
// Trigger starts a workflow for events of one type from one source whose
|
|
// fields match its filter.
|
|
type Trigger struct {
|
|
OrgID string `db:"org_id"`
|
|
Name string `db:"name"`
|
|
Source string `db:"source"`
|
|
EventType string `db:"event_type"`
|
|
Filter jsontext.Value `db:"filter"`
|
|
Workflow string `db:"workflow"`
|
|
Enabled bool `db:"enabled"`
|
|
// ScheduleEverySeconds is the interval a scheduled trigger fires on; it
|
|
// is null for a trigger that fires on events.
|
|
ScheduleEverySeconds sql.NullInt64 `db:"schedule_every_s"`
|
|
// ScheduleSince is when the schedule began or last changed.
|
|
ScheduleSince sql.NullTime `db:"schedule_since"`
|
|
}
|
|
|
|
// Store is the events module's Postgres store.
|
|
type Store struct {
|
|
db *sqlx.DB
|
|
}
|
|
|
|
// New returns a store on db.
|
|
func New(db *sqlx.DB) *Store { return &Store{db: db} }
|
|
|
|
// Source returns the named source.
|
|
func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) {
|
|
var src Source
|
|
err := s.db.GetContext(ctx, &src, `
|
|
SELECT org_id, name, connector, token_sha256, key_fields FROM events.sources
|
|
WHERE org_id = $1 AND name = $2`, orgID, name)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Source{}, ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return Source{}, fmt.Errorf("read source: %w", err)
|
|
}
|
|
return src, nil
|
|
}
|
|
|
|
// Sources returns every source of the organisation.
|
|
func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) {
|
|
var out []Source
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT org_id, name, connector, token_sha256, cursor, key_fields FROM events.sources
|
|
WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
|
|
return nil, fmt.Errorf("list sources: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// SetCursor records where a listening source resumes.
|
|
func (s *Store) SetCursor(ctx context.Context, orgID, name, cursor string) error {
|
|
if _, err := s.db.ExecContext(ctx, `
|
|
UPDATE events.sources SET cursor = $3, updated_at = now() WHERE org_id = $1 AND name = $2`,
|
|
orgID, name, cursor); err != nil {
|
|
return fmt.Errorf("save cursor: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpsertSource creates a source or replaces its connector and token.
|
|
func (s *Store) UpsertSource(ctx context.Context, src Source) error {
|
|
if _, err := s.db.NamedExecContext(ctx, `
|
|
INSERT INTO events.sources (org_id, name, connector, token_sha256)
|
|
VALUES (:org_id, :name, :connector, :token_sha256)
|
|
ON CONFLICT (org_id, name) DO UPDATE
|
|
SET connector = EXCLUDED.connector, token_sha256 = EXCLUDED.token_sha256, updated_at = now()`, src); err != nil {
|
|
return fmt.Errorf("save source: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Insert stores an event unless its source already delivered the same source
|
|
// event id. It reports whether the event was new.
|
|
func (s *Store) Insert(ctx context.Context, e Event) (bool, error) {
|
|
res, err := s.db.NamedExecContext(ctx, `
|
|
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, body, occurred_at,
|
|
own, origin_run_id, group_key, group_filter)
|
|
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :body, :occurred_at,
|
|
:own, :origin_run_id, :group_key, :group_filter)
|
|
ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e)
|
|
if err != nil {
|
|
return false, fmt.Errorf("store event: %w", err)
|
|
}
|
|
n, err := res.RowsAffected()
|
|
if err != nil {
|
|
return false, fmt.Errorf("store event: %w", err)
|
|
}
|
|
return n == 1, nil
|
|
}
|
|
|
|
// Dispatch claims the oldest undispatched event and hands it to fn inside one
|
|
// transaction. fn's matched result and the dispatched mark commit together,
|
|
// so an event is dispatched once even with several workers. It reports
|
|
// whether there was an event to dispatch.
|
|
func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (matched bool, err error)) (bool, error) {
|
|
tx, err := s.db.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return false, fmt.Errorf("begin dispatch: %w", err)
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
var e Event
|
|
err = tx.GetContext(ctx, &e, `
|
|
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
|
|
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
|
|
FROM events.events WHERE dispatched_at IS NULL
|
|
ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, fmt.Errorf("claim event: %w", err)
|
|
}
|
|
matched, err := fn(tx, e)
|
|
if err != nil {
|
|
return true, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `
|
|
UPDATE events.events SET dispatched_at = now(), matched = $2 WHERE id = $1`, e.ID, matched); err != nil {
|
|
return true, fmt.Errorf("mark dispatched: %w", err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return true, fmt.Errorf("commit dispatch: %w", err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Link points e at the earliest event its source delivered before it with the
|
|
// same link key, and reports whether it found one. Only an earlier event
|
|
// counts: a firing alert must not link to the resolve that follows it.
|
|
func Link(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) {
|
|
if !e.LinkKey.Valid {
|
|
return false, nil
|
|
}
|
|
const earlier = `org_id = $2 AND source = $3 AND link_key = $4 AND id <> $1
|
|
AND (received_at, id) < ($5, $1)`
|
|
res, err := tx.ExecContext(ctx, `
|
|
UPDATE events.events SET linked_to = (
|
|
SELECT id FROM events.events WHERE `+earlier+`
|
|
ORDER BY received_at, id LIMIT 1)
|
|
WHERE id = $1 AND EXISTS (SELECT 1 FROM events.events WHERE `+earlier+`)`,
|
|
e.ID, e.OrgID, e.Source, e.LinkKey.String, e.ReceivedAt)
|
|
if err != nil {
|
|
return false, fmt.Errorf("link event: %w", err)
|
|
}
|
|
n, err := res.RowsAffected()
|
|
return n == 1, err
|
|
}
|
|
|
|
// Triggers returns the enabled triggers for one source and event type.
|
|
func Triggers(ctx context.Context, tx *sqlx.Tx, orgID, source, eventType string) ([]Trigger, error) {
|
|
var out []Trigger
|
|
if err := tx.SelectContext(ctx, &out, `
|
|
SELECT org_id, name, source, event_type, filter, workflow, enabled FROM events.triggers
|
|
WHERE org_id = $1 AND source = $2 AND event_type = $3 AND enabled ORDER BY name`,
|
|
orgID, source, eventType); err != nil {
|
|
return nil, fmt.Errorf("read triggers: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// RecordDispatch notes that trigger started run for an event.
|
|
func RecordDispatch(ctx context.Context, tx *sqlx.Tx, eventID uuid.UUID, trigger string, runID uuid.UUID) error {
|
|
if _, err := tx.ExecContext(ctx, `
|
|
INSERT INTO events.dispatches (event_id, trigger, run_id) VALUES ($1, $2, $3)
|
|
ON CONFLICT (event_id, trigger) DO NOTHING`, eventID, trigger, runID); err != nil {
|
|
return fmt.Errorf("record dispatch: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpsertTrigger creates or replaces a trigger.
|
|
func (s *Store) UpsertTrigger(ctx context.Context, t Trigger) error {
|
|
if _, err := s.db.NamedExecContext(ctx, `
|
|
INSERT INTO events.triggers (org_id, name, source, event_type, filter, workflow, enabled,
|
|
schedule_every_s, schedule_since)
|
|
VALUES (:org_id, :name, :source, :event_type, :filter, :workflow, :enabled,
|
|
CAST(:schedule_every_s AS integer), CASE WHEN CAST(:schedule_every_s AS integer) IS NULL THEN NULL ELSE now() END)
|
|
ON CONFLICT (org_id, name) DO UPDATE
|
|
SET source = EXCLUDED.source, event_type = EXCLUDED.event_type, filter = EXCLUDED.filter,
|
|
workflow = EXCLUDED.workflow, enabled = EXCLUDED.enabled,
|
|
schedule_since = CASE
|
|
WHEN EXCLUDED.schedule_every_s IS NULL THEN NULL
|
|
WHEN events.triggers.schedule_every_s IS NOT DISTINCT FROM EXCLUDED.schedule_every_s
|
|
THEN events.triggers.schedule_since
|
|
ELSE now() END,
|
|
schedule_every_s = EXCLUDED.schedule_every_s, updated_at = now()`, t); err != nil {
|
|
return fmt.Errorf("save trigger: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ScheduledTriggers returns the enabled triggers that fire on an interval.
|
|
func (s *Store) ScheduledTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
|
|
var out []Trigger
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
|
|
FROM events.triggers
|
|
WHERE org_id = $1 AND enabled AND schedule_every_s IS NOT NULL ORDER BY name`, orgID); err != nil {
|
|
return nil, fmt.Errorf("read schedules: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// EventFilter narrows the events a list returns. A zero field doesn't filter,
|
|
// and Matched distinguishes events a trigger started a run for from the rest.
|
|
type EventFilter struct {
|
|
Source string
|
|
Type string
|
|
Matched *bool
|
|
Limit int
|
|
}
|
|
|
|
// Recent returns an organisation's newest events that match filter.
|
|
func (s *Store) Recent(ctx context.Context, orgID string, f EventFilter) ([]Event, error) {
|
|
var out []Event
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
|
|
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
|
|
FROM events.events
|
|
WHERE org_id = $1
|
|
AND ($2 = '' OR source = $2)
|
|
AND ($3 = '' OR type = $3)
|
|
AND ($4::boolean IS NULL OR matched = $4)
|
|
ORDER BY received_at DESC LIMIT $5`, orgID, f.Source, f.Type, f.Matched, f.Limit); err != nil {
|
|
return nil, fmt.Errorf("list events: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Event returns one event with the payload it arrived in.
|
|
func (s *Store) Event(ctx context.Context, orgID string, id uuid.UUID) (Event, error) {
|
|
var e Event
|
|
err := s.db.GetContext(ctx, &e, `
|
|
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, body, occurred_at,
|
|
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
|
|
FROM events.events WHERE org_id = $1 AND id = $2`, orgID, id)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Event{}, ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return Event{}, fmt.Errorf("read event: %w", err)
|
|
}
|
|
return e, nil
|
|
}
|
|
|
|
// TriggerRun is a run a trigger started for an event.
|
|
type TriggerRun struct {
|
|
Trigger string `db:"trigger"`
|
|
RunID uuid.UUID `db:"run_id"`
|
|
}
|
|
|
|
// RunsForEvent returns the runs triggers started for an event.
|
|
func (s *Store) RunsForEvent(ctx context.Context, orgID string, id uuid.UUID) ([]TriggerRun, error) {
|
|
var out []TriggerRun
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT d.trigger, d.run_id FROM events.dispatches d
|
|
JOIN events.events e ON e.id = d.event_id
|
|
WHERE e.org_id = $1 AND d.event_id = $2 ORDER BY d.trigger`, orgID, id); err != nil {
|
|
return nil, fmt.Errorf("read event runs: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// AllTriggers returns every trigger of the organisation, enabled or not.
|
|
func (s *Store) AllTriggers(ctx context.Context, orgID string) ([]Trigger, error) {
|
|
var out []Trigger
|
|
if err := s.db.SelectContext(ctx, &out, `
|
|
SELECT org_id, name, source, event_type, filter, workflow, enabled, schedule_every_s, schedule_since
|
|
FROM events.triggers WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
|
|
return nil, fmt.Errorf("list triggers: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Origin is the event and trigger that started a run.
|
|
type Origin struct {
|
|
EventID uuid.UUID `db:"event_id"`
|
|
Source string `db:"source"`
|
|
Type string `db:"type"`
|
|
Fields jsontext.Value `db:"fields"`
|
|
ReceivedAt time.Time `db:"received_at"`
|
|
Trigger string `db:"trigger"`
|
|
Filter jsontext.Value `db:"filter"`
|
|
}
|
|
|
|
// OriginOf returns what started a run; ok is false when no trigger did.
|
|
func (s *Store) OriginOf(ctx context.Context, orgID string, runID uuid.UUID) (Origin, bool, error) {
|
|
var o Origin
|
|
err := s.db.GetContext(ctx, &o, `
|
|
SELECT e.id AS event_id, e.source, e.type, e.fields, e.received_at, d.trigger,
|
|
COALESCE(t.filter, '{}') AS filter
|
|
FROM events.dispatches d
|
|
JOIN events.events e ON e.id = d.event_id
|
|
LEFT JOIN events.triggers t ON t.org_id = e.org_id AND t.name = d.trigger
|
|
WHERE e.org_id = $1 AND d.run_id = $2`, orgID, runID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Origin{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Origin{}, false, fmt.Errorf("read run origin: %w", err)
|
|
}
|
|
return o, true, nil
|
|
}
|