feat(events): unmatched events are grouped by source, type and key fields, and Archie's own deliveries never group (#1994)

This commit is contained in:
sam committed 2026-11-16 09:42:23 +11:00
1 parent 0c8b673615
commit 50b75f6987
14 files changed
+547 -25

No files matched your search

+1 -1
View File
@@ -95,7 +95,7 @@ func Run(ctx context.Context, configPath string) error {
classes.Routes(mux, guard, allow)
policies := newPolicies(db, log, mux, guard)
// An answer a channel delivers is an event, and settles its request.
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow, tell)
hear.Routes(mux, guard)
judge := newJudgement(db, log, conns, classes, policies, settings, hear)
engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, memory.NewModule(db, orgID))) //nolint:contextcheck // see above
@@ -0,0 +1,43 @@
-- +goose Up
-- Provenance and the group key are fixed when an event arrives. own marks an
-- event that is the echo of a delivery Archie sent, linked to the run that
-- sent it; such an event never forms or joins a group. The key is the source,
-- the type and the source's key fields.
ALTER TABLE events.events
ADD COLUMN own boolean NOT NULL DEFAULT false,
ADD COLUMN origin_run_id uuid,
ADD COLUMN group_key text,
ADD COLUMN group_filter jsonb;
-- A source's key fields are dotted paths into its events' fields; an object
-- stands for its scalar members. Empty means the default list.
ALTER TABLE events.sources ADD COLUMN key_fields jsonb NOT NULL DEFAULT '[]';
-- A group is the events no trigger matched that share a key. Its source, type
-- and filter are the trigger it proposes.
CREATE TABLE events.groups (
id uuid PRIMARY KEY,
org_id text NOT NULL,
key text NOT NULL,
source text NOT NULL,
event_type text NOT NULL,
filter jsonb NOT NULL,
count bigint NOT NULL,
first_seen timestamptz NOT NULL,
last_seen timestamptz NOT NULL,
UNIQUE (org_id, key)
);
-- The deliveries module recognises an echo by the id its channel gave the
-- message.
CREATE INDEX messages_external ON deliveries.messages (org_id, channel, external_id) WHERE external_id IS NOT NULL;
-- +goose Down
DROP INDEX deliveries.messages_external;
DROP TABLE events.groups;
ALTER TABLE events.sources DROP COLUMN key_fields;
ALTER TABLE events.events
DROP COLUMN group_filter,
DROP COLUMN group_key,
DROP COLUMN origin_run_id,
DROP COLUMN own;
+13
View File
@@ -15,6 +15,7 @@ import (
"fmt"
"log/slog"
"net/http"
"slices"
"strconv"
"time"
@@ -163,6 +164,18 @@ func (m *Module) send(ctx context.Context, msg store.Message) error {
return nil
}
// SentBy reports whether channel confirmed a message under one of ids, the id
// a source's event carries when it echoes what Archie sent, and the run that
// sent it (nil when the message belonged to none).
func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uuid.UUID, bool, error) {
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { return id == "" })
if len(ids) == 0 {
return uuid.Nil, false, nil
}
run, found, err := m.store.SentBy(ctx, m.orgID, channel, ids)
return run.UUID, found, err
}
// Work sends queued messages to their connectors until ctx ends. A failed send
// is tried again after a backoff that grows to maxBackoff, never dropped.
func (m *Module) Work(ctx context.Context) {
@@ -136,6 +136,22 @@ func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error {
return nil
}
// SentBy finds a message its channel confirmed under one of ids. The run is
// the one that sent it, nil when the message belonged to none.
func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) (uuid.NullUUID, bool, error) {
var run uuid.NullUUID
err := s.db.GetContext(ctx, &run, `
SELECT run_id FROM deliveries.messages
WHERE org_id = $1 AND channel = $2 AND external_id = ANY($3) LIMIT 1`, orgID, channel, ids)
if errors.Is(err, sql.ErrNoRows) {
return uuid.NullUUID{}, false, nil
}
if err != nil {
return uuid.NullUUID{}, false, fmt.Errorf("find sent message: %w", err)
}
return run, true, nil
}
// Failed records why an attempt failed; the lease already set when it is
// tried again.
func (s *Store) Failed(ctx context.Context, seq int64, reason string) error {
+100 -11
View File
@@ -45,6 +45,13 @@ type Runs interface {
Start(ctx context.Context, req runs.StartRequest) (uuid.UUID, error)
}
// Deliveries recognises the echo of a message Archie sent. It returns the run
// that sent the message the ids name on channel, which is nil when the message
// belonged to no run.
type Deliveries interface {
SentBy(ctx context.Context, channel string, ids ...string) (run uuid.UUID, found bool, err error)
}
// Answers settles the approval requests answer events name.
type Answers interface {
Settle(ctx context.Context, id uuid.UUID, identity, via string, approved bool) (bool, error)
@@ -64,20 +71,23 @@ type Module struct {
connectors Connectors
runs Runs
answers Answers
deliveries Deliveries
log *slog.Logger
poll time.Duration
}
// NewModule returns the events module for orgID.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers) *Module {
return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, log: log, poll: time.Second}
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers, d Deliveries) *Module {
return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
}
// Routes adds the webhook intake and the guarded admin routes.
func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) {
mux.HandleFunc("POST /hooks/{source}", m.intake)
mux.Handle("PUT /api/sources/{name}", guard(http.HandlerFunc(m.putSource)))
mux.Handle("PUT /api/sources/{name}/key-fields", guard(http.HandlerFunc(m.putKeyFields)))
mux.Handle("PUT /api/triggers/{name}", guard(http.HandlerFunc(m.putTrigger)))
mux.Handle("GET /api/groups", guard(http.HandlerFunc(m.groups)))
mux.Handle("GET /api/events", guard(http.HandlerFunc(m.recent)))
}
@@ -110,7 +120,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
stored, duplicates, err := m.save(r.Context(), src.Name, parsed)
stored, duplicates, err := m.save(r.Context(), src, parsed)
if err != nil {
httpjson.Error(w, http.StatusServiceUnavailable, err.Error())
return
@@ -118,15 +128,15 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
}
func (m *Module) save(ctx context.Context, source string, parsed []connectors.Event) (stored, duplicates int, err error) {
// save stores each event, recording at arrival whether it echoes a delivery
// Archie sent, so those never reach grouping.
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event) (stored, duplicates int, err error) {
for _, e := range parsed {
row := store.Event{
ID: uuid.New(), OrgID: m.orgID, Source: source, Type: e.Type, SourceEventID: e.SourceEventID,
Fields: e.Fields,
run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey)
if err != nil {
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
}
row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != ""
row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero()
isNew, err := m.store.Insert(ctx, row)
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, own, run))
if err != nil {
return stored, duplicates, err
}
@@ -201,7 +211,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) {
err = fmt.Errorf("connector %q isn't running", src.Connector)
} else {
err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error {
if _, _, err := m.save(ctx, src.Name, events); err != nil {
if _, _, err := m.save(ctx, src, events); err != nil {
return err
}
if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil {
@@ -250,6 +260,10 @@ func (m *Module) answer(ctx context.Context, e store.Event) error {
// dispatch never starts a second run.
func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
ctx := context.Background()
if e.Own {
// Archie's own message coming back starts nothing and proposes nothing.
return false, nil
}
if e.Type == answerType {
return false, m.answer(ctx, e)
}
@@ -282,6 +296,15 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID)
matched = true
}
if !matched {
created, err := store.NoteUnmatched(ctx, tx, e)
if err != nil {
return false, err
}
if created {
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
}
}
return matched, nil
}
@@ -388,6 +411,72 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
httpjson.Write(w, http.StatusOK, map[string]string{"trigger": name})
}
type keyFieldsRequest struct {
Paths []string `json:"paths"`
}
// putKeyFields sets the fields a source's events are grouped by. It applies to
// events that arrive after it.
func (m *Module) putKeyFields(w http.ResponseWriter, r *http.Request) {
var req keyFieldsRequest
if err := httpjson.Read(r, &req); err != nil {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
if req.Paths == nil {
req.Paths = []string{}
}
raw, err := json.Marshal(req.Paths)
if err != nil {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
err = m.store.SetKeyFields(r.Context(), m.orgID, r.PathValue("name"), raw)
if errors.Is(err, store.ErrNotFound) {
httpjson.Error(w, http.StatusNotFound, "no such source")
return
}
if err != nil {
httpjson.Error(w, http.StatusInternalServerError, err.Error())
return
}
httpjson.Write(w, http.StatusOK, map[string][]string{"key_fields": req.Paths})
}
// groupView is a group of unmatched events as the API returns it. Proposed is
// the trigger it suggests, with the workflow left for the person to choose.
type groupView struct {
ID uuid.UUID `json:"id"`
Count int64 `json:"count"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
Proposed proposedTrigger `json:"proposed"`
}
type proposedTrigger struct {
Source string `json:"source"`
EventType string `json:"event_type"`
Filter jsontext.Value `json:"filter"`
Workflow string `json:"workflow"`
}
func (m *Module) groups(w http.ResponseWriter, r *http.Request) {
groups, err := m.store.Groups(r.Context(), m.orgID, 200)
if err != nil {
m.log.ErrorContext(r.Context(), "list groups", "error", err)
httpjson.Error(w, http.StatusInternalServerError, "couldn't list groups")
return
}
out := make([]groupView, 0, len(groups))
for _, g := range groups {
out = append(out, groupView{
ID: g.ID, Count: g.Count, FirstSeen: g.FirstSeen, LastSeen: g.LastSeen,
Proposed: proposedTrigger{Source: g.Source, EventType: g.EventType, Filter: g.Filter},
})
}
httpjson.Write(w, http.StatusOK, out)
}
func (m *Module) recent(w http.ResponseWriter, r *http.Request) {
limit := 50
if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 500 {
+77
View File
@@ -0,0 +1,77 @@
package events
import (
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json/jsontext"
json "encoding/json/v2"
"slices"
"strings"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/match"
"github.com/samcharles93/archie-core/modules/events/internal/store"
)
// defaultKeyFields name the fields that identify what an event is about, for
// a source that lists none. An object stands for its scalar members, so
// "labels" covers every label.
var defaultKeyFields = []string{"alertname", "name", "labels", "severity", "service", "host", "instance"}
// groupKey is the identity events of one kind share: source, type and the
// values at keyFields. It also returns those values as the filter of the
// trigger the group proposes.
func groupKey(source, eventType string, fields jsontext.Value, keyFields []string) (string, map[string]string) {
if len(keyFields) == 0 {
keyFields = defaultKeyFields
}
filter := match.Pick(fields, keyFields)
paths := make([]string, 0, len(filter))
for p := range filter {
paths = append(paths, p)
}
slices.Sort(paths)
var b strings.Builder
b.WriteString(source + "\x00" + eventType)
for _, p := range paths {
b.WriteString("\x00" + p + "=" + filter[p])
}
sum := sha256.Sum256([]byte(b.String()))
return hex.EncodeToString(sum[:]), filter
}
// keyPaths returns the source's key fields; none means the default list.
func keyPaths(src store.Source) []string {
var paths []string
if len(src.KeyFields) > 0 {
// Only a list that encodes is stored, so a failure reads as none.
_ = json.Unmarshal(src.KeyFields, &paths)
}
return paths
}
// newRow returns the row stored for an event a source delivered. An echo of a
// delivery Archie sent (own) is recorded with the run that sent it and gets no
// group key, so it can never form or join a group. Any other event's key is
// fixed here, once, from the source's key fields.
func newRow(orgID string, src store.Source, e connectors.Event, own bool, runID uuid.UUID) store.Event {
row := store.Event{
ID: uuid.New(), OrgID: orgID, Source: src.Name, Type: e.Type, SourceEventID: e.SourceEventID,
Fields: e.Fields, Own: own,
}
row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != ""
row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero()
if own {
row.OriginRunID = uuid.NullUUID{UUID: runID, Valid: runID != uuid.Nil}
return row
}
key, filter := groupKey(src.Name, e.Type, e.Fields, keyPaths(src))
// A map of strings always encodes.
raw, _ := json.Marshal(filter)
row.GroupKey = sql.NullString{String: key, Valid: true}
row.GroupFilter = raw
return row
}
+35
View File
@@ -0,0 +1,35 @@
package events
import (
"testing"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/events/internal/store"
)
// TestNewRowGrouping guards two rules: an echo of Archie's own delivery never
// carries a group key, so it can't become a proposal, and events differing
// only outside the key fields share one key.
func TestNewRowGrouping(t *testing.T) {
src := store.Source{Name: "alerts", KeyFields: []byte(`["alertname","labels"]`)}
run := uuid.New()
ev := func(fields string) connectors.Event {
return connectors.Event{Type: "alert.firing", SourceEventID: fields, Fields: []byte(fields)}
}
disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), false, uuid.Nil)
again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), false, uuid.Nil)
other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), false, uuid.Nil)
own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), true, run)
if !disk.GroupKey.Valid || disk.GroupKey != again.GroupKey {
t.Errorf("same key fields gave different keys: %q, %q", disk.GroupKey.String, again.GroupKey.String)
}
if disk.GroupKey == other.GroupKey {
t.Error("different labels shared a key")
}
if own.GroupKey.Valid || own.GroupFilter != nil || !own.Own || own.OriginRunID.UUID != run {
t.Errorf("own delivery was keyed or not linked to its run: %+v", own)
}
}
+42 -6
View File
@@ -19,25 +19,61 @@ func Fields(fields jsontext.Value, filter map[string]string) bool {
return false
}
for path, want := range filter {
got, ok := lookup(doc, strings.Split(path, "."))
if !ok || got != want {
node, ok := walk(doc, strings.Split(path, "."))
if !ok {
return false
}
if got, ok := scalar(node); !ok || got != want {
return false
}
}
return true
}
func lookup(doc any, path []string) (string, bool) {
// Pick returns the event's values at paths, keyed by dotted path. A path that
// names an object stands for its scalar members, and a path the event doesn't
// have is left out. The result is a filter Fields accepts for this event.
func Pick(fields jsontext.Value, paths []string) map[string]string {
out := map[string]string{}
var doc any
if err := json.Unmarshal(fields, &doc); err != nil {
return out
}
for _, p := range paths {
node, ok := walk(doc, strings.Split(p, "."))
if !ok {
continue
}
if obj, isObj := node.(map[string]any); isObj {
for k, member := range obj {
if v, ok := scalar(member); ok {
out[p+"."+k] = v
}
}
continue
}
if v, ok := scalar(node); ok {
out[p] = v
}
}
return out
}
func walk(doc any, path []string) (any, bool) {
for _, key := range path {
obj, ok := doc.(map[string]any)
if !ok {
return "", false
return nil, false
}
if doc, ok = obj[key]; !ok {
return "", false
return nil, false
}
}
switch v := doc.(type) {
return doc, true
}
func scalar(v any) (string, bool) {
switch v := v.(type) {
case string:
return v, true
case float64, bool:
+77
View File
@@ -0,0 +1,77 @@
package store
import (
"context"
"encoding/json/jsontext"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
)
// SetKeyFields replaces the paths a source's events are grouped by.
func (s *Store) SetKeyFields(ctx context.Context, orgID, name string, paths jsontext.Value) error {
res, err := s.db.ExecContext(ctx, `
UPDATE events.sources SET key_fields = $3, updated_at = now() WHERE org_id = $1 AND name = $2`,
orgID, name, paths)
if err != nil {
return fmt.Errorf("save key fields: %w", err)
}
if n, err := res.RowsAffected(); err != nil || n == 0 {
return ErrNotFound
}
return nil
}
// Group is the events no trigger matched that share a key. Its source, type
// and filter are the trigger it proposes.
type Group struct {
ID uuid.UUID `db:"id"`
Source string `db:"source"`
EventType string `db:"event_type"`
Filter jsontext.Value `db:"filter"`
Count int64 `db:"count"`
FirstSeen time.Time `db:"first_seen"`
LastSeen time.Time `db:"last_seen"`
}
// NoteUnmatched adds an event no trigger matched to the group its key names,
// starting the group if it is the first, and reports whether it did. An event
// with no key (a schedule tick, which has none to give) is left out.
func NoteUnmatched(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) {
if !e.GroupKey.Valid {
return false, nil
}
var created bool
err := tx.QueryRowxContext(ctx, `
INSERT INTO events.groups (id, org_id, key, source, event_type, filter, count, first_seen, last_seen)
VALUES ($1, $2, $3, $4, $5, $6, 1, $7, $7)
ON CONFLICT (org_id, key) DO UPDATE
SET count = events.groups.count + 1,
first_seen = LEAST(events.groups.first_seen, EXCLUDED.first_seen),
last_seen = GREATEST(events.groups.last_seen, EXCLUDED.last_seen)
RETURNING xmax = 0`,
uuid.New(), e.OrgID, e.GroupKey.String, e.Source, e.Type, e.GroupFilter, e.ReceivedAt).Scan(&created)
if err != nil {
return false, fmt.Errorf("group event: %w", err)
}
return created, nil
}
// Groups returns the groups no trigger covers, newest activity first. A group
// stops being listed once a trigger exists for its source, type and filter.
func (s *Store) Groups(ctx context.Context, orgID string, limit int) ([]Group, error) {
var out []Group
if err := s.db.SelectContext(ctx, &out, `
SELECT g.id, g.source, g.event_type, g.filter, g.count, g.first_seen, g.last_seen
FROM events.groups g
WHERE g.org_id = $1 AND NOT EXISTS (
SELECT 1 FROM events.triggers t
WHERE t.org_id = g.org_id AND t.source = g.source AND t.event_type = g.event_type
AND t.filter = g.filter)
ORDER BY g.last_seen DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list groups: %w", err)
}
return out, nil
}
+16 -6
View File
@@ -26,6 +26,8 @@ type Source struct {
TokenSHA256 string `db:"token_sha256"`
// Cursor is where a listening source resumes.
Cursor string `db:"cursor"`
// KeyFields are the paths its events are grouped by; none means the default.
KeyFields jsontext.Value `db:"key_fields"`
}
// Event is one stored event.
@@ -42,6 +44,12 @@ type Event struct {
ReceivedAt time.Time `db:"received_at"`
Matched bool `db:"matched"`
DispatchedAt sql.NullTime `db:"dispatched_at"`
// Own marks the echo of a delivery Archie sent, and OriginRunID is the run
// that sent it. An own event has no group.
Own bool `db:"own"`
OriginRunID uuid.NullUUID `db:"origin_run_id"`
GroupKey sql.NullString `db:"group_key"`
GroupFilter jsontext.Value `db:"group_filter"`
}
// Trigger starts a workflow for events of one type from one source whose
@@ -73,7 +81,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} }
func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) {
var src Source
err := s.db.GetContext(ctx, &src, `
SELECT org_id, name, connector, token_sha256 FROM events.sources
SELECT org_id, name, connector, token_sha256, key_fields FROM events.sources
WHERE org_id = $1 AND name = $2`, orgID, name)
if errors.Is(err, sql.ErrNoRows) {
return Source{}, ErrNotFound
@@ -88,7 +96,7 @@ func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error)
func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) {
var out []Source
if err := s.db.SelectContext(ctx, &out, `
SELECT org_id, name, connector, token_sha256, cursor FROM events.sources
SELECT org_id, name, connector, token_sha256, cursor, key_fields FROM events.sources
WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
return nil, fmt.Errorf("list sources: %w", err)
}
@@ -121,8 +129,10 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error {
// event id. It reports whether the event was new.
func (s *Store) Insert(ctx context.Context, e Event) (bool, error) {
res, err := s.db.NamedExecContext(ctx, `
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at)
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at)
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at,
own, origin_run_id, group_key, group_filter)
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at,
:own, :origin_run_id, :group_key, :group_filter)
ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e)
if err != nil {
return false, fmt.Errorf("store event: %w", err)
@@ -147,7 +157,7 @@ func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (mat
var e Event
err = tx.GetContext(ctx, &e, `
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
received_at, matched, dispatched_at
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
FROM events.events WHERE dispatched_at IS NULL
ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`)
if errors.Is(err, sql.ErrNoRows) {
@@ -252,7 +262,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, e
var out []Event
if err := s.db.SelectContext(ctx, &out, `
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
received_at, matched, dispatched_at
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
FROM events.events WHERE org_id = $1 ORDER BY received_at DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list events: %w", err)
}
+1
View File
@@ -34,6 +34,7 @@ function signOut() {
<nav v-if="signedIn" class="flex flex-1 items-center gap-3 text-label">
<RouterLink to="/composer" active-class="text-foreground" class="text-muted-foreground">Composer</RouterLink>
<RouterLink to="/runs" active-class="text-foreground" class="text-muted-foreground">Runs</RouterLink>
<RouterLink to="/groups" active-class="text-foreground" class="text-muted-foreground">Unmatched</RouterLink>
<RouterLink to="/classes" active-class="text-foreground" class="text-muted-foreground">Classes</RouterLink>
<RouterLink to="/people" active-class="text-foreground" class="text-muted-foreground">People</RouterLink>
<RouterLink to="/policies" active-class="text-foreground" class="text-muted-foreground">Policies</RouterLink>
+110
View File
@@ -0,0 +1,110 @@
<script setup lang="ts">
import { onBeforeUnmount, onMounted, ref } from "vue";
import { Button } from "@/components/ui/button";
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
import { Input } from "@/components/ui/input";
import { ReadError } from "@/components/ui/read-error";
import { Spinner } from "@/components/ui/spinner";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { Timestamp } from "@/components/ui/timestamp";
import { listGroups, saveTrigger, Unauthorized, type EventGroup } from "./api";
const emit = defineEmits<{ unauthorized: [] }>();
const groups = ref<EventGroup[]>([]);
const workflows = ref<Record<string, string>>({});
const loading = ref(true);
const creating = ref<string | null>(null);
const error = ref("");
const actionError = ref("");
const controller = new AbortController();
async function load() {
loading.value = true;
error.value = "";
try {
groups.value = await listGroups(controller.signal);
} catch (err) {
fail(err, (message) => (error.value = message));
} finally {
loading.value = false;
}
}
/** A trigger name unique to the group, so creating never replaces another trigger. */
function triggerName(g: EventGroup): string {
const base = `${g.proposed.source}-${g.proposed.event_type}`.toLowerCase().replace(/[^a-z0-9]+/g, "-");
return `${base.slice(0, 50)}-${g.id.slice(0, 8)}`.replace(/^[^a-z]+/, "g");
}
async function create(g: EventGroup) {
const workflow = (workflows.value[g.id] ?? "").trim();
if (!workflow) return;
creating.value = g.id;
actionError.value = "";
try {
await saveTrigger(triggerName(g), { ...g.proposed, workflow });
groups.value = await listGroups(controller.signal);
} catch (err) {
fail(err, (message) => (actionError.value = message));
} finally {
creating.value = null;
}
}
function fail(err: unknown, set: (message: string) => void) {
if (controller.signal.aborted) return;
if (err instanceof Unauthorized) emit("unauthorized");
set(err instanceof Error ? err.message : String(err));
}
const filterText = (filter: Record<string, string>) =>
Object.entries(filter)
.map(([k, v]) => `${k}=${v}`)
.join(" ");
onMounted(load);
onBeforeUnmount(() => controller.abort());
</script>
<template>
<div>
<Spinner v-if="loading" />
<ReadError v-else-if="error" :message="error" retry-label="Retry" @retry="load" />
<Empty v-else-if="groups.length === 0">
<EmptyHeader><EmptyTitle>No unmatched events</EmptyTitle></EmptyHeader>
</Empty>
<template v-else>
<p v-if="actionError" class="mb-2 text-caption text-danger">{{ actionError }}</p>
<Table>
<TableHeader>
<TableRow>
<TableHead>Trigger</TableHead>
<TableHead>Events</TableHead>
<TableHead>Last seen</TableHead>
<TableHead>Workflow</TableHead>
</TableRow>
</TableHeader>
<TableBody>
<TableRow v-for="g in groups" :key="g.id">
<TableCell>
<div>{{ g.proposed.source }} {{ g.proposed.event_type }}</div>
<div class="font-mono text-caption text-muted-foreground">{{ filterText(g.proposed.filter) }}</div>
</TableCell>
<TableCell>{{ g.count }}</TableCell>
<TableCell><Timestamp :at="g.last_seen" /></TableCell>
<TableCell>
<form class="flex gap-2" @submit.prevent="create(g)">
<Input v-model="workflows[g.id]" aria-label="Workflow" />
<Button type="submit" size="xs" :disabled="creating === g.id || !(workflows[g.id] ?? '').trim()">
Create
</Button>
</form>
</TableCell>
</TableRow>
</TableBody>
</Table>
</template>
</div>
</template>
+15 -1
View File
@@ -312,7 +312,21 @@ export async function resetClass(id: string): Promise<void> {
}
}
export const listRuns = (signal?: AbortSignal) => get<Run[]>("/api/runs", signal);
/** Unmatched events that share a source, type and key fields, with the trigger they propose. */
export interface EventGroup {
id: string;
count: number;
first_seen: string;
last_seen: string;
proposed: { source: string; event_type: string; filter: Record<string, string>; workflow: string };
}
export const listGroups = (signal?: AbortSignal) => get<EventGroup[]>("/api/groups", signal);
export const saveTrigger = (name: string, trigger: EventGroup["proposed"]) =>
put(`/api/triggers/${encodeURIComponent(name)}`, trigger);
export const listRuns =(signal?: AbortSignal) => get<Run[]>("/api/runs", signal);
export const runTrail = (id: string, signal?: AbortSignal) => get<Trail>(`/api/runs/${encodeURIComponent(id)}`, signal);
/**
+1
View File
@@ -6,6 +6,7 @@ export default createRouter({
{ path: "/", redirect: "/composer" },
{ path: "/composer", name: "composer", component: () => import("./ComposerView.vue") },
{ path: "/runs", name: "runs", component: () => import("./RunsView.vue") },
{ path: "/groups", name: "groups", component: () => import("./GroupsView.vue") },
{ path: "/classes", name: "classes", component: () => import("./ClassesView.vue") },
{ path: "/classes/:id", name: "class", component: () => import("./ClassView.vue"), props: true },
{ path: "/people", name: "people", component: () => import("./PeopleView.vue") },