diff --git a/internal/app/archie/run.go b/internal/app/archie/run.go index b63036b17..e21a0e3a4 100644 --- a/internal/app/archie/run.go +++ b/internal/app/archie/run.go @@ -95,7 +95,7 @@ func Run(ctx context.Context, configPath string) error { classes.Routes(mux, guard, allow) policies := newPolicies(db, log, mux, guard) // An answer a channel delivers is an event, and settles its request. - hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow) + hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow, tell) hear.Routes(mux, guard) judge := newJudgement(db, log, conns, classes, policies, settings, hear) engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, memory.NewModule(db, orgID))) //nolint:contextcheck // see above diff --git a/internal/platform/pg/migrations/00029_events_groups.sql b/internal/platform/pg/migrations/00029_events_groups.sql new file mode 100644 index 000000000..3b6867be0 --- /dev/null +++ b/internal/platform/pg/migrations/00029_events_groups.sql @@ -0,0 +1,43 @@ +-- +goose Up +-- Provenance and the group key are fixed when an event arrives. own marks an +-- event that is the echo of a delivery Archie sent, linked to the run that +-- sent it; such an event never forms or joins a group. The key is the source, +-- the type and the source's key fields. +ALTER TABLE events.events + ADD COLUMN own boolean NOT NULL DEFAULT false, + ADD COLUMN origin_run_id uuid, + ADD COLUMN group_key text, + ADD COLUMN group_filter jsonb; + +-- A source's key fields are dotted paths into its events' fields; an object +-- stands for its scalar members. Empty means the default list. +ALTER TABLE events.sources ADD COLUMN key_fields jsonb NOT NULL DEFAULT '[]'; + +-- A group is the events no trigger matched that share a key. Its source, type +-- and filter are the trigger it proposes. +CREATE TABLE events.groups ( + id uuid PRIMARY KEY, + org_id text NOT NULL, + key text NOT NULL, + source text NOT NULL, + event_type text NOT NULL, + filter jsonb NOT NULL, + count bigint NOT NULL, + first_seen timestamptz NOT NULL, + last_seen timestamptz NOT NULL, + UNIQUE (org_id, key) +); + +-- The deliveries module recognises an echo by the id its channel gave the +-- message. +CREATE INDEX messages_external ON deliveries.messages (org_id, channel, external_id) WHERE external_id IS NOT NULL; + +-- +goose Down +DROP INDEX deliveries.messages_external; +DROP TABLE events.groups; +ALTER TABLE events.sources DROP COLUMN key_fields; +ALTER TABLE events.events + DROP COLUMN group_filter, + DROP COLUMN group_key, + DROP COLUMN origin_run_id, + DROP COLUMN own; diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index b69acb390..d1b0c574b 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -15,6 +15,7 @@ import ( "fmt" "log/slog" "net/http" + "slices" "strconv" "time" @@ -163,6 +164,18 @@ func (m *Module) send(ctx context.Context, msg store.Message) error { return nil } +// SentBy reports whether channel confirmed a message under one of ids, the id +// a source's event carries when it echoes what Archie sent, and the run that +// sent it (nil when the message belonged to none). +func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uuid.UUID, bool, error) { + ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { return id == "" }) + if len(ids) == 0 { + return uuid.Nil, false, nil + } + run, found, err := m.store.SentBy(ctx, m.orgID, channel, ids) + return run.UUID, found, err +} + // Work sends queued messages to their connectors until ctx ends. A failed send // is tried again after a backoff that grows to maxBackoff, never dropped. func (m *Module) Work(ctx context.Context) { diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index d75f06452..b95ca7838 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -136,6 +136,22 @@ func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { return nil } +// SentBy finds a message its channel confirmed under one of ids. The run is +// the one that sent it, nil when the message belonged to none. +func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) (uuid.NullUUID, bool, error) { + var run uuid.NullUUID + err := s.db.GetContext(ctx, &run, ` + SELECT run_id FROM deliveries.messages + WHERE org_id = $1 AND channel = $2 AND external_id = ANY($3) LIMIT 1`, orgID, channel, ids) + if errors.Is(err, sql.ErrNoRows) { + return uuid.NullUUID{}, false, nil + } + if err != nil { + return uuid.NullUUID{}, false, fmt.Errorf("find sent message: %w", err) + } + return run, true, nil +} + // Failed records why an attempt failed; the lease already set when it is // tried again. func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { diff --git a/modules/events/events.go b/modules/events/events.go index 31ecf9b5c..1caf3108a 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -45,6 +45,13 @@ type Runs interface { Start(ctx context.Context, req runs.StartRequest) (uuid.UUID, error) } +// Deliveries recognises the echo of a message Archie sent. It returns the run +// that sent the message the ids name on channel, which is nil when the message +// belonged to no run. +type Deliveries interface { + SentBy(ctx context.Context, channel string, ids ...string) (run uuid.UUID, found bool, err error) +} + // Answers settles the approval requests answer events name. type Answers interface { Settle(ctx context.Context, id uuid.UUID, identity, via string, approved bool) (bool, error) @@ -64,20 +71,23 @@ type Module struct { connectors Connectors runs Runs answers Answers + deliveries Deliveries log *slog.Logger poll time.Duration } // NewModule returns the events module for orgID. -func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers) *Module { - return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, log: log, poll: time.Second} +func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers, d Deliveries) *Module { + return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second} } // Routes adds the webhook intake and the guarded admin routes. func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) { mux.HandleFunc("POST /hooks/{source}", m.intake) mux.Handle("PUT /api/sources/{name}", guard(http.HandlerFunc(m.putSource))) + mux.Handle("PUT /api/sources/{name}/key-fields", guard(http.HandlerFunc(m.putKeyFields))) mux.Handle("PUT /api/triggers/{name}", guard(http.HandlerFunc(m.putTrigger))) + mux.Handle("GET /api/groups", guard(http.HandlerFunc(m.groups))) mux.Handle("GET /api/events", guard(http.HandlerFunc(m.recent))) } @@ -110,7 +120,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusBadRequest, err.Error()) return } - stored, duplicates, err := m.save(r.Context(), src.Name, parsed) + stored, duplicates, err := m.save(r.Context(), src, parsed) if err != nil { httpjson.Error(w, http.StatusServiceUnavailable, err.Error()) return @@ -118,15 +128,15 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates}) } -func (m *Module) save(ctx context.Context, source string, parsed []connectors.Event) (stored, duplicates int, err error) { +// save stores each event, recording at arrival whether it echoes a delivery +// Archie sent, so those never reach grouping. +func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event) (stored, duplicates int, err error) { for _, e := range parsed { - row := store.Event{ - ID: uuid.New(), OrgID: m.orgID, Source: source, Type: e.Type, SourceEventID: e.SourceEventID, - Fields: e.Fields, + run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey) + if err != nil { + return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err) } - row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != "" - row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero() - isNew, err := m.store.Insert(ctx, row) + isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, own, run)) if err != nil { return stored, duplicates, err } @@ -201,7 +211,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) { err = fmt.Errorf("connector %q isn't running", src.Connector) } else { err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error { - if _, _, err := m.save(ctx, src.Name, events); err != nil { + if _, _, err := m.save(ctx, src, events); err != nil { return err } if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil { @@ -250,6 +260,10 @@ func (m *Module) answer(ctx context.Context, e store.Event) error { // dispatch never starts a second run. func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { ctx := context.Background() + if e.Own { + // Archie's own message coming back starts nothing and proposes nothing. + return false, nil + } if e.Type == answerType { return false, m.answer(ctx, e) } @@ -282,6 +296,15 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID) matched = true } + if !matched { + created, err := store.NoteUnmatched(ctx, tx, e) + if err != nil { + return false, err + } + if created { + m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type) + } + } return matched, nil } @@ -388,6 +411,72 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) { httpjson.Write(w, http.StatusOK, map[string]string{"trigger": name}) } +type keyFieldsRequest struct { + Paths []string `json:"paths"` +} + +// putKeyFields sets the fields a source's events are grouped by. It applies to +// events that arrive after it. +func (m *Module) putKeyFields(w http.ResponseWriter, r *http.Request) { + var req keyFieldsRequest + if err := httpjson.Read(r, &req); err != nil { + httpjson.Error(w, http.StatusBadRequest, err.Error()) + return + } + if req.Paths == nil { + req.Paths = []string{} + } + raw, err := json.Marshal(req.Paths) + if err != nil { + httpjson.Error(w, http.StatusBadRequest, err.Error()) + return + } + err = m.store.SetKeyFields(r.Context(), m.orgID, r.PathValue("name"), raw) + if errors.Is(err, store.ErrNotFound) { + httpjson.Error(w, http.StatusNotFound, "no such source") + return + } + if err != nil { + httpjson.Error(w, http.StatusInternalServerError, err.Error()) + return + } + httpjson.Write(w, http.StatusOK, map[string][]string{"key_fields": req.Paths}) +} + +// groupView is a group of unmatched events as the API returns it. Proposed is +// the trigger it suggests, with the workflow left for the person to choose. +type groupView struct { + ID uuid.UUID `json:"id"` + Count int64 `json:"count"` + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + Proposed proposedTrigger `json:"proposed"` +} + +type proposedTrigger struct { + Source string `json:"source"` + EventType string `json:"event_type"` + Filter jsontext.Value `json:"filter"` + Workflow string `json:"workflow"` +} + +func (m *Module) groups(w http.ResponseWriter, r *http.Request) { + groups, err := m.store.Groups(r.Context(), m.orgID, 200) + if err != nil { + m.log.ErrorContext(r.Context(), "list groups", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't list groups") + return + } + out := make([]groupView, 0, len(groups)) + for _, g := range groups { + out = append(out, groupView{ + ID: g.ID, Count: g.Count, FirstSeen: g.FirstSeen, LastSeen: g.LastSeen, + Proposed: proposedTrigger{Source: g.Source, EventType: g.EventType, Filter: g.Filter}, + }) + } + httpjson.Write(w, http.StatusOK, out) +} + func (m *Module) recent(w http.ResponseWriter, r *http.Request) { limit := 50 if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 500 { diff --git a/modules/events/grouping.go b/modules/events/grouping.go new file mode 100644 index 000000000..d8c5589e2 --- /dev/null +++ b/modules/events/grouping.go @@ -0,0 +1,77 @@ +package events + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json/jsontext" + json "encoding/json/v2" + "slices" + "strings" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/match" + "github.com/samcharles93/archie-core/modules/events/internal/store" +) + +// defaultKeyFields name the fields that identify what an event is about, for +// a source that lists none. An object stands for its scalar members, so +// "labels" covers every label. +var defaultKeyFields = []string{"alertname", "name", "labels", "severity", "service", "host", "instance"} + +// groupKey is the identity events of one kind share: source, type and the +// values at keyFields. It also returns those values as the filter of the +// trigger the group proposes. +func groupKey(source, eventType string, fields jsontext.Value, keyFields []string) (string, map[string]string) { + if len(keyFields) == 0 { + keyFields = defaultKeyFields + } + filter := match.Pick(fields, keyFields) + paths := make([]string, 0, len(filter)) + for p := range filter { + paths = append(paths, p) + } + slices.Sort(paths) + var b strings.Builder + b.WriteString(source + "\x00" + eventType) + for _, p := range paths { + b.WriteString("\x00" + p + "=" + filter[p]) + } + sum := sha256.Sum256([]byte(b.String())) + return hex.EncodeToString(sum[:]), filter +} + +// keyPaths returns the source's key fields; none means the default list. +func keyPaths(src store.Source) []string { + var paths []string + if len(src.KeyFields) > 0 { + // Only a list that encodes is stored, so a failure reads as none. + _ = json.Unmarshal(src.KeyFields, &paths) + } + return paths +} + +// newRow returns the row stored for an event a source delivered. An echo of a +// delivery Archie sent (own) is recorded with the run that sent it and gets no +// group key, so it can never form or join a group. Any other event's key is +// fixed here, once, from the source's key fields. +func newRow(orgID string, src store.Source, e connectors.Event, own bool, runID uuid.UUID) store.Event { + row := store.Event{ + ID: uuid.New(), OrgID: orgID, Source: src.Name, Type: e.Type, SourceEventID: e.SourceEventID, + Fields: e.Fields, Own: own, + } + row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != "" + row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero() + if own { + row.OriginRunID = uuid.NullUUID{UUID: runID, Valid: runID != uuid.Nil} + return row + } + key, filter := groupKey(src.Name, e.Type, e.Fields, keyPaths(src)) + // A map of strings always encodes. + raw, _ := json.Marshal(filter) + row.GroupKey = sql.NullString{String: key, Valid: true} + row.GroupFilter = raw + return row +} diff --git a/modules/events/grouping_test.go b/modules/events/grouping_test.go new file mode 100644 index 000000000..5cf192b71 --- /dev/null +++ b/modules/events/grouping_test.go @@ -0,0 +1,35 @@ +package events + +import ( + "testing" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/events/internal/store" +) + +// TestNewRowGrouping guards two rules: an echo of Archie's own delivery never +// carries a group key, so it can't become a proposal, and events differing +// only outside the key fields share one key. +func TestNewRowGrouping(t *testing.T) { + src := store.Source{Name: "alerts", KeyFields: []byte(`["alertname","labels"]`)} + run := uuid.New() + ev := func(fields string) connectors.Event { + return connectors.Event{Type: "alert.firing", SourceEventID: fields, Fields: []byte(fields)} + } + disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), false, uuid.Nil) + again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), false, uuid.Nil) + other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), false, uuid.Nil) + own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), true, run) + + if !disk.GroupKey.Valid || disk.GroupKey != again.GroupKey { + t.Errorf("same key fields gave different keys: %q, %q", disk.GroupKey.String, again.GroupKey.String) + } + if disk.GroupKey == other.GroupKey { + t.Error("different labels shared a key") + } + if own.GroupKey.Valid || own.GroupFilter != nil || !own.Own || own.OriginRunID.UUID != run { + t.Errorf("own delivery was keyed or not linked to its run: %+v", own) + } +} diff --git a/modules/events/internal/match/match.go b/modules/events/internal/match/match.go index c085d656f..eaa870f1b 100644 --- a/modules/events/internal/match/match.go +++ b/modules/events/internal/match/match.go @@ -19,25 +19,61 @@ func Fields(fields jsontext.Value, filter map[string]string) bool { return false } for path, want := range filter { - got, ok := lookup(doc, strings.Split(path, ".")) - if !ok || got != want { + node, ok := walk(doc, strings.Split(path, ".")) + if !ok { + return false + } + if got, ok := scalar(node); !ok || got != want { return false } } return true } -func lookup(doc any, path []string) (string, bool) { +// Pick returns the event's values at paths, keyed by dotted path. A path that +// names an object stands for its scalar members, and a path the event doesn't +// have is left out. The result is a filter Fields accepts for this event. +func Pick(fields jsontext.Value, paths []string) map[string]string { + out := map[string]string{} + var doc any + if err := json.Unmarshal(fields, &doc); err != nil { + return out + } + for _, p := range paths { + node, ok := walk(doc, strings.Split(p, ".")) + if !ok { + continue + } + if obj, isObj := node.(map[string]any); isObj { + for k, member := range obj { + if v, ok := scalar(member); ok { + out[p+"."+k] = v + } + } + continue + } + if v, ok := scalar(node); ok { + out[p] = v + } + } + return out +} + +func walk(doc any, path []string) (any, bool) { for _, key := range path { obj, ok := doc.(map[string]any) if !ok { - return "", false + return nil, false } if doc, ok = obj[key]; !ok { - return "", false + return nil, false } } - switch v := doc.(type) { + return doc, true +} + +func scalar(v any) (string, bool) { + switch v := v.(type) { case string: return v, true case float64, bool: diff --git a/modules/events/internal/store/groups.go b/modules/events/internal/store/groups.go new file mode 100644 index 000000000..bf17ba97e --- /dev/null +++ b/modules/events/internal/store/groups.go @@ -0,0 +1,77 @@ +package store + +import ( + "context" + "encoding/json/jsontext" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" +) + +// SetKeyFields replaces the paths a source's events are grouped by. +func (s *Store) SetKeyFields(ctx context.Context, orgID, name string, paths jsontext.Value) error { + res, err := s.db.ExecContext(ctx, ` + UPDATE events.sources SET key_fields = $3, updated_at = now() WHERE org_id = $1 AND name = $2`, + orgID, name, paths) + if err != nil { + return fmt.Errorf("save key fields: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return ErrNotFound + } + return nil +} + +// Group is the events no trigger matched that share a key. Its source, type +// and filter are the trigger it proposes. +type Group struct { + ID uuid.UUID `db:"id"` + Source string `db:"source"` + EventType string `db:"event_type"` + Filter jsontext.Value `db:"filter"` + Count int64 `db:"count"` + FirstSeen time.Time `db:"first_seen"` + LastSeen time.Time `db:"last_seen"` +} + +// NoteUnmatched adds an event no trigger matched to the group its key names, +// starting the group if it is the first, and reports whether it did. An event +// with no key (a schedule tick, which has none to give) is left out. +func NoteUnmatched(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) { + if !e.GroupKey.Valid { + return false, nil + } + var created bool + err := tx.QueryRowxContext(ctx, ` + INSERT INTO events.groups (id, org_id, key, source, event_type, filter, count, first_seen, last_seen) + VALUES ($1, $2, $3, $4, $5, $6, 1, $7, $7) + ON CONFLICT (org_id, key) DO UPDATE + SET count = events.groups.count + 1, + first_seen = LEAST(events.groups.first_seen, EXCLUDED.first_seen), + last_seen = GREATEST(events.groups.last_seen, EXCLUDED.last_seen) + RETURNING xmax = 0`, + uuid.New(), e.OrgID, e.GroupKey.String, e.Source, e.Type, e.GroupFilter, e.ReceivedAt).Scan(&created) + if err != nil { + return false, fmt.Errorf("group event: %w", err) + } + return created, nil +} + +// Groups returns the groups no trigger covers, newest activity first. A group +// stops being listed once a trigger exists for its source, type and filter. +func (s *Store) Groups(ctx context.Context, orgID string, limit int) ([]Group, error) { + var out []Group + if err := s.db.SelectContext(ctx, &out, ` + SELECT g.id, g.source, g.event_type, g.filter, g.count, g.first_seen, g.last_seen + FROM events.groups g + WHERE g.org_id = $1 AND NOT EXISTS ( + SELECT 1 FROM events.triggers t + WHERE t.org_id = g.org_id AND t.source = g.source AND t.event_type = g.event_type + AND t.filter = g.filter) + ORDER BY g.last_seen DESC LIMIT $2`, orgID, limit); err != nil { + return nil, fmt.Errorf("list groups: %w", err) + } + return out, nil +} diff --git a/modules/events/internal/store/store.go b/modules/events/internal/store/store.go index c97573380..8e0deda49 100644 --- a/modules/events/internal/store/store.go +++ b/modules/events/internal/store/store.go @@ -26,6 +26,8 @@ type Source struct { TokenSHA256 string `db:"token_sha256"` // Cursor is where a listening source resumes. Cursor string `db:"cursor"` + // KeyFields are the paths its events are grouped by; none means the default. + KeyFields jsontext.Value `db:"key_fields"` } // Event is one stored event. @@ -42,6 +44,12 @@ type Event struct { ReceivedAt time.Time `db:"received_at"` Matched bool `db:"matched"` DispatchedAt sql.NullTime `db:"dispatched_at"` + // Own marks the echo of a delivery Archie sent, and OriginRunID is the run + // that sent it. An own event has no group. + Own bool `db:"own"` + OriginRunID uuid.NullUUID `db:"origin_run_id"` + GroupKey sql.NullString `db:"group_key"` + GroupFilter jsontext.Value `db:"group_filter"` } // Trigger starts a workflow for events of one type from one source whose @@ -73,7 +81,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} } func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) { var src Source err := s.db.GetContext(ctx, &src, ` - SELECT org_id, name, connector, token_sha256 FROM events.sources + SELECT org_id, name, connector, token_sha256, key_fields FROM events.sources WHERE org_id = $1 AND name = $2`, orgID, name) if errors.Is(err, sql.ErrNoRows) { return Source{}, ErrNotFound @@ -88,7 +96,7 @@ func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) { var out []Source if err := s.db.SelectContext(ctx, &out, ` - SELECT org_id, name, connector, token_sha256, cursor FROM events.sources + SELECT org_id, name, connector, token_sha256, cursor, key_fields FROM events.sources WHERE org_id = $1 ORDER BY name`, orgID); err != nil { return nil, fmt.Errorf("list sources: %w", err) } @@ -121,8 +129,10 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error { // event id. It reports whether the event was new. func (s *Store) Insert(ctx context.Context, e Event) (bool, error) { res, err := s.db.NamedExecContext(ctx, ` - INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at) - VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at) + INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at, + own, origin_run_id, group_key, group_filter) + VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at, + :own, :origin_run_id, :group_key, :group_filter) ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e) if err != nil { return false, fmt.Errorf("store event: %w", err) @@ -147,7 +157,7 @@ func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (mat var e Event err = tx.GetContext(ctx, &e, ` SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at, - received_at, matched, dispatched_at + received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter FROM events.events WHERE dispatched_at IS NULL ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`) if errors.Is(err, sql.ErrNoRows) { @@ -252,7 +262,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, e var out []Event if err := s.db.SelectContext(ctx, &out, ` SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at, - received_at, matched, dispatched_at + received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter FROM events.events WHERE org_id = $1 ORDER BY received_at DESC LIMIT $2`, orgID, limit); err != nil { return nil, fmt.Errorf("list events: %w", err) } diff --git a/ui/src/archie/App.vue b/ui/src/archie/App.vue index 9af0db196..3b5aa7a89 100644 --- a/ui/src/archie/App.vue +++ b/ui/src/archie/App.vue @@ -34,6 +34,7 @@ function signOut() {