feat(events): unmatched events are grouped by source, type and key fields, and Archie's own deliveries never group (#1994)
This commit is contained in:
14 files changed
+547
-25
No files matched your search
@@ -95,7 +95,7 @@ func Run(ctx context.Context, configPath string) error {
|
||||
classes.Routes(mux, guard, allow)
|
||||
policies := newPolicies(db, log, mux, guard)
|
||||
// An answer a channel delivers is an event, and settles its request.
|
||||
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
|
||||
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow, tell)
|
||||
hear.Routes(mux, guard)
|
||||
judge := newJudgement(db, log, conns, classes, policies, settings, hear)
|
||||
engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, memory.NewModule(db, orgID))) //nolint:contextcheck // see above
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
-- +goose Up
|
||||
-- Provenance and the group key are fixed when an event arrives. own marks an
|
||||
-- event that is the echo of a delivery Archie sent, linked to the run that
|
||||
-- sent it; such an event never forms or joins a group. The key is the source,
|
||||
-- the type and the source's key fields.
|
||||
ALTER TABLE events.events
|
||||
ADD COLUMN own boolean NOT NULL DEFAULT false,
|
||||
ADD COLUMN origin_run_id uuid,
|
||||
ADD COLUMN group_key text,
|
||||
ADD COLUMN group_filter jsonb;
|
||||
|
||||
-- A source's key fields are dotted paths into its events' fields; an object
|
||||
-- stands for its scalar members. Empty means the default list.
|
||||
ALTER TABLE events.sources ADD COLUMN key_fields jsonb NOT NULL DEFAULT '[]';
|
||||
|
||||
-- A group is the events no trigger matched that share a key. Its source, type
|
||||
-- and filter are the trigger it proposes.
|
||||
CREATE TABLE events.groups (
|
||||
id uuid PRIMARY KEY,
|
||||
org_id text NOT NULL,
|
||||
key text NOT NULL,
|
||||
source text NOT NULL,
|
||||
event_type text NOT NULL,
|
||||
filter jsonb NOT NULL,
|
||||
count bigint NOT NULL,
|
||||
first_seen timestamptz NOT NULL,
|
||||
last_seen timestamptz NOT NULL,
|
||||
UNIQUE (org_id, key)
|
||||
);
|
||||
|
||||
-- The deliveries module recognises an echo by the id its channel gave the
|
||||
-- message.
|
||||
CREATE INDEX messages_external ON deliveries.messages (org_id, channel, external_id) WHERE external_id IS NOT NULL;
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX deliveries.messages_external;
|
||||
DROP TABLE events.groups;
|
||||
ALTER TABLE events.sources DROP COLUMN key_fields;
|
||||
ALTER TABLE events.events
|
||||
DROP COLUMN group_filter,
|
||||
DROP COLUMN group_key,
|
||||
DROP COLUMN origin_run_id,
|
||||
DROP COLUMN own;
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -163,6 +164,18 @@ func (m *Module) send(ctx context.Context, msg store.Message) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SentBy reports whether channel confirmed a message under one of ids, the id
|
||||
// a source's event carries when it echoes what Archie sent, and the run that
|
||||
// sent it (nil when the message belonged to none).
|
||||
func (m *Module) SentBy(ctx context.Context, channel string, ids ...string) (uuid.UUID, bool, error) {
|
||||
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool { return id == "" })
|
||||
if len(ids) == 0 {
|
||||
return uuid.Nil, false, nil
|
||||
}
|
||||
run, found, err := m.store.SentBy(ctx, m.orgID, channel, ids)
|
||||
return run.UUID, found, err
|
||||
}
|
||||
|
||||
// Work sends queued messages to their connectors until ctx ends. A failed send
|
||||
// is tried again after a backoff that grows to maxBackoff, never dropped.
|
||||
func (m *Module) Work(ctx context.Context) {
|
||||
|
||||
@@ -136,6 +136,22 @@ func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SentBy finds a message its channel confirmed under one of ids. The run is
|
||||
// the one that sent it, nil when the message belonged to none.
|
||||
func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) (uuid.NullUUID, bool, error) {
|
||||
var run uuid.NullUUID
|
||||
err := s.db.GetContext(ctx, &run, `
|
||||
SELECT run_id FROM deliveries.messages
|
||||
WHERE org_id = $1 AND channel = $2 AND external_id = ANY($3) LIMIT 1`, orgID, channel, ids)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return uuid.NullUUID{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return uuid.NullUUID{}, false, fmt.Errorf("find sent message: %w", err)
|
||||
}
|
||||
return run, true, nil
|
||||
}
|
||||
|
||||
// Failed records why an attempt failed; the lease already set when it is
|
||||
// tried again.
|
||||
func (s *Store) Failed(ctx context.Context, seq int64, reason string) error {
|
||||
|
||||
+100
-11
@@ -45,6 +45,13 @@ type Runs interface {
|
||||
Start(ctx context.Context, req runs.StartRequest) (uuid.UUID, error)
|
||||
}
|
||||
|
||||
// Deliveries recognises the echo of a message Archie sent. It returns the run
|
||||
// that sent the message the ids name on channel, which is nil when the message
|
||||
// belonged to no run.
|
||||
type Deliveries interface {
|
||||
SentBy(ctx context.Context, channel string, ids ...string) (run uuid.UUID, found bool, err error)
|
||||
}
|
||||
|
||||
// Answers settles the approval requests answer events name.
|
||||
type Answers interface {
|
||||
Settle(ctx context.Context, id uuid.UUID, identity, via string, approved bool) (bool, error)
|
||||
@@ -64,20 +71,23 @@ type Module struct {
|
||||
connectors Connectors
|
||||
runs Runs
|
||||
answers Answers
|
||||
deliveries Deliveries
|
||||
log *slog.Logger
|
||||
poll time.Duration
|
||||
}
|
||||
|
||||
// NewModule returns the events module for orgID.
|
||||
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers) *Module {
|
||||
return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, log: log, poll: time.Second}
|
||||
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers, d Deliveries) *Module {
|
||||
return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, deliveries: d, log: log, poll: time.Second}
|
||||
}
|
||||
|
||||
// Routes adds the webhook intake and the guarded admin routes.
|
||||
func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) {
|
||||
mux.HandleFunc("POST /hooks/{source}", m.intake)
|
||||
mux.Handle("PUT /api/sources/{name}", guard(http.HandlerFunc(m.putSource)))
|
||||
mux.Handle("PUT /api/sources/{name}/key-fields", guard(http.HandlerFunc(m.putKeyFields)))
|
||||
mux.Handle("PUT /api/triggers/{name}", guard(http.HandlerFunc(m.putTrigger)))
|
||||
mux.Handle("GET /api/groups", guard(http.HandlerFunc(m.groups)))
|
||||
mux.Handle("GET /api/events", guard(http.HandlerFunc(m.recent)))
|
||||
}
|
||||
|
||||
@@ -110,7 +120,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
stored, duplicates, err := m.save(r.Context(), src.Name, parsed)
|
||||
stored, duplicates, err := m.save(r.Context(), src, parsed)
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusServiceUnavailable, err.Error())
|
||||
return
|
||||
@@ -118,15 +128,15 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
|
||||
}
|
||||
|
||||
func (m *Module) save(ctx context.Context, source string, parsed []connectors.Event) (stored, duplicates int, err error) {
|
||||
// save stores each event, recording at arrival whether it echoes a delivery
|
||||
// Archie sent, so those never reach grouping.
|
||||
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event) (stored, duplicates int, err error) {
|
||||
for _, e := range parsed {
|
||||
row := store.Event{
|
||||
ID: uuid.New(), OrgID: m.orgID, Source: source, Type: e.Type, SourceEventID: e.SourceEventID,
|
||||
Fields: e.Fields,
|
||||
run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey)
|
||||
if err != nil {
|
||||
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
|
||||
}
|
||||
row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != ""
|
||||
row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero()
|
||||
isNew, err := m.store.Insert(ctx, row)
|
||||
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, own, run))
|
||||
if err != nil {
|
||||
return stored, duplicates, err
|
||||
}
|
||||
@@ -201,7 +211,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) {
|
||||
err = fmt.Errorf("connector %q isn't running", src.Connector)
|
||||
} else {
|
||||
err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error {
|
||||
if _, _, err := m.save(ctx, src.Name, events); err != nil {
|
||||
if _, _, err := m.save(ctx, src, events); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil {
|
||||
@@ -250,6 +260,10 @@ func (m *Module) answer(ctx context.Context, e store.Event) error {
|
||||
// dispatch never starts a second run.
|
||||
func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
|
||||
ctx := context.Background()
|
||||
if e.Own {
|
||||
// Archie's own message coming back starts nothing and proposes nothing.
|
||||
return false, nil
|
||||
}
|
||||
if e.Type == answerType {
|
||||
return false, m.answer(ctx, e)
|
||||
}
|
||||
@@ -282,6 +296,15 @@ func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) {
|
||||
m.log.InfoContext(ctx, "trigger started run", "event", e.ID, "trigger", t.Name, "run", runID)
|
||||
matched = true
|
||||
}
|
||||
if !matched {
|
||||
created, err := store.NoteUnmatched(ctx, tx, e)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if created {
|
||||
m.log.InfoContext(ctx, "unmatched events started a group", "source", e.Source, "type", e.Type)
|
||||
}
|
||||
}
|
||||
return matched, nil
|
||||
}
|
||||
|
||||
@@ -388,6 +411,72 @@ func (m *Module) putTrigger(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Write(w, http.StatusOK, map[string]string{"trigger": name})
|
||||
}
|
||||
|
||||
type keyFieldsRequest struct {
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
|
||||
// putKeyFields sets the fields a source's events are grouped by. It applies to
|
||||
// events that arrive after it.
|
||||
func (m *Module) putKeyFields(w http.ResponseWriter, r *http.Request) {
|
||||
var req keyFieldsRequest
|
||||
if err := httpjson.Read(r, &req); err != nil {
|
||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if req.Paths == nil {
|
||||
req.Paths = []string{}
|
||||
}
|
||||
raw, err := json.Marshal(req.Paths)
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
err = m.store.SetKeyFields(r.Context(), m.orgID, r.PathValue("name"), raw)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
httpjson.Error(w, http.StatusNotFound, "no such source")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
httpjson.Write(w, http.StatusOK, map[string][]string{"key_fields": req.Paths})
|
||||
}
|
||||
|
||||
// groupView is a group of unmatched events as the API returns it. Proposed is
|
||||
// the trigger it suggests, with the workflow left for the person to choose.
|
||||
type groupView struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
Count int64 `json:"count"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
Proposed proposedTrigger `json:"proposed"`
|
||||
}
|
||||
|
||||
type proposedTrigger struct {
|
||||
Source string `json:"source"`
|
||||
EventType string `json:"event_type"`
|
||||
Filter jsontext.Value `json:"filter"`
|
||||
Workflow string `json:"workflow"`
|
||||
}
|
||||
|
||||
func (m *Module) groups(w http.ResponseWriter, r *http.Request) {
|
||||
groups, err := m.store.Groups(r.Context(), m.orgID, 200)
|
||||
if err != nil {
|
||||
m.log.ErrorContext(r.Context(), "list groups", "error", err)
|
||||
httpjson.Error(w, http.StatusInternalServerError, "couldn't list groups")
|
||||
return
|
||||
}
|
||||
out := make([]groupView, 0, len(groups))
|
||||
for _, g := range groups {
|
||||
out = append(out, groupView{
|
||||
ID: g.ID, Count: g.Count, FirstSeen: g.FirstSeen, LastSeen: g.LastSeen,
|
||||
Proposed: proposedTrigger{Source: g.Source, EventType: g.EventType, Filter: g.Filter},
|
||||
})
|
||||
}
|
||||
httpjson.Write(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (m *Module) recent(w http.ResponseWriter, r *http.Request) {
|
||||
limit := 50
|
||||
if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 500 {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/match"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
)
|
||||
|
||||
// defaultKeyFields name the fields that identify what an event is about, for
|
||||
// a source that lists none. An object stands for its scalar members, so
|
||||
// "labels" covers every label.
|
||||
var defaultKeyFields = []string{"alertname", "name", "labels", "severity", "service", "host", "instance"}
|
||||
|
||||
// groupKey is the identity events of one kind share: source, type and the
|
||||
// values at keyFields. It also returns those values as the filter of the
|
||||
// trigger the group proposes.
|
||||
func groupKey(source, eventType string, fields jsontext.Value, keyFields []string) (string, map[string]string) {
|
||||
if len(keyFields) == 0 {
|
||||
keyFields = defaultKeyFields
|
||||
}
|
||||
filter := match.Pick(fields, keyFields)
|
||||
paths := make([]string, 0, len(filter))
|
||||
for p := range filter {
|
||||
paths = append(paths, p)
|
||||
}
|
||||
slices.Sort(paths)
|
||||
var b strings.Builder
|
||||
b.WriteString(source + "\x00" + eventType)
|
||||
for _, p := range paths {
|
||||
b.WriteString("\x00" + p + "=" + filter[p])
|
||||
}
|
||||
sum := sha256.Sum256([]byte(b.String()))
|
||||
return hex.EncodeToString(sum[:]), filter
|
||||
}
|
||||
|
||||
// keyPaths returns the source's key fields; none means the default list.
|
||||
func keyPaths(src store.Source) []string {
|
||||
var paths []string
|
||||
if len(src.KeyFields) > 0 {
|
||||
// Only a list that encodes is stored, so a failure reads as none.
|
||||
_ = json.Unmarshal(src.KeyFields, &paths)
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
// newRow returns the row stored for an event a source delivered. An echo of a
|
||||
// delivery Archie sent (own) is recorded with the run that sent it and gets no
|
||||
// group key, so it can never form or join a group. Any other event's key is
|
||||
// fixed here, once, from the source's key fields.
|
||||
func newRow(orgID string, src store.Source, e connectors.Event, own bool, runID uuid.UUID) store.Event {
|
||||
row := store.Event{
|
||||
ID: uuid.New(), OrgID: orgID, Source: src.Name, Type: e.Type, SourceEventID: e.SourceEventID,
|
||||
Fields: e.Fields, Own: own,
|
||||
}
|
||||
row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != ""
|
||||
row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero()
|
||||
if own {
|
||||
row.OriginRunID = uuid.NullUUID{UUID: runID, Valid: runID != uuid.Nil}
|
||||
return row
|
||||
}
|
||||
key, filter := groupKey(src.Name, e.Type, e.Fields, keyPaths(src))
|
||||
// A map of strings always encodes.
|
||||
raw, _ := json.Marshal(filter)
|
||||
row.GroupKey = sql.NullString{String: key, Valid: true}
|
||||
row.GroupFilter = raw
|
||||
return row
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/events/internal/store"
|
||||
)
|
||||
|
||||
// TestNewRowGrouping guards two rules: an echo of Archie's own delivery never
|
||||
// carries a group key, so it can't become a proposal, and events differing
|
||||
// only outside the key fields share one key.
|
||||
func TestNewRowGrouping(t *testing.T) {
|
||||
src := store.Source{Name: "alerts", KeyFields: []byte(`["alertname","labels"]`)}
|
||||
run := uuid.New()
|
||||
ev := func(fields string) connectors.Event {
|
||||
return connectors.Event{Type: "alert.firing", SourceEventID: fields, Fields: []byte(fields)}
|
||||
}
|
||||
disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), false, uuid.Nil)
|
||||
again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), false, uuid.Nil)
|
||||
other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), false, uuid.Nil)
|
||||
own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), true, run)
|
||||
|
||||
if !disk.GroupKey.Valid || disk.GroupKey != again.GroupKey {
|
||||
t.Errorf("same key fields gave different keys: %q, %q", disk.GroupKey.String, again.GroupKey.String)
|
||||
}
|
||||
if disk.GroupKey == other.GroupKey {
|
||||
t.Error("different labels shared a key")
|
||||
}
|
||||
if own.GroupKey.Valid || own.GroupFilter != nil || !own.Own || own.OriginRunID.UUID != run {
|
||||
t.Errorf("own delivery was keyed or not linked to its run: %+v", own)
|
||||
}
|
||||
}
|
||||
@@ -19,25 +19,61 @@ func Fields(fields jsontext.Value, filter map[string]string) bool {
|
||||
return false
|
||||
}
|
||||
for path, want := range filter {
|
||||
got, ok := lookup(doc, strings.Split(path, "."))
|
||||
if !ok || got != want {
|
||||
node, ok := walk(doc, strings.Split(path, "."))
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if got, ok := scalar(node); !ok || got != want {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func lookup(doc any, path []string) (string, bool) {
|
||||
// Pick returns the event's values at paths, keyed by dotted path. A path that
|
||||
// names an object stands for its scalar members, and a path the event doesn't
|
||||
// have is left out. The result is a filter Fields accepts for this event.
|
||||
func Pick(fields jsontext.Value, paths []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
var doc any
|
||||
if err := json.Unmarshal(fields, &doc); err != nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range paths {
|
||||
node, ok := walk(doc, strings.Split(p, "."))
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if obj, isObj := node.(map[string]any); isObj {
|
||||
for k, member := range obj {
|
||||
if v, ok := scalar(member); ok {
|
||||
out[p+"."+k] = v
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if v, ok := scalar(node); ok {
|
||||
out[p] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func walk(doc any, path []string) (any, bool) {
|
||||
for _, key := range path {
|
||||
obj, ok := doc.(map[string]any)
|
||||
if !ok {
|
||||
return "", false
|
||||
return nil, false
|
||||
}
|
||||
if doc, ok = obj[key]; !ok {
|
||||
return "", false
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
switch v := doc.(type) {
|
||||
return doc, true
|
||||
}
|
||||
|
||||
func scalar(v any) (string, bool) {
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
return v, true
|
||||
case float64, bool:
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
// SetKeyFields replaces the paths a source's events are grouped by.
|
||||
func (s *Store) SetKeyFields(ctx context.Context, orgID, name string, paths jsontext.Value) error {
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
UPDATE events.sources SET key_fields = $3, updated_at = now() WHERE org_id = $1 AND name = $2`,
|
||||
orgID, name, paths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("save key fields: %w", err)
|
||||
}
|
||||
if n, err := res.RowsAffected(); err != nil || n == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Group is the events no trigger matched that share a key. Its source, type
|
||||
// and filter are the trigger it proposes.
|
||||
type Group struct {
|
||||
ID uuid.UUID `db:"id"`
|
||||
Source string `db:"source"`
|
||||
EventType string `db:"event_type"`
|
||||
Filter jsontext.Value `db:"filter"`
|
||||
Count int64 `db:"count"`
|
||||
FirstSeen time.Time `db:"first_seen"`
|
||||
LastSeen time.Time `db:"last_seen"`
|
||||
}
|
||||
|
||||
// NoteUnmatched adds an event no trigger matched to the group its key names,
|
||||
// starting the group if it is the first, and reports whether it did. An event
|
||||
// with no key (a schedule tick, which has none to give) is left out.
|
||||
func NoteUnmatched(ctx context.Context, tx *sqlx.Tx, e Event) (bool, error) {
|
||||
if !e.GroupKey.Valid {
|
||||
return false, nil
|
||||
}
|
||||
var created bool
|
||||
err := tx.QueryRowxContext(ctx, `
|
||||
INSERT INTO events.groups (id, org_id, key, source, event_type, filter, count, first_seen, last_seen)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 1, $7, $7)
|
||||
ON CONFLICT (org_id, key) DO UPDATE
|
||||
SET count = events.groups.count + 1,
|
||||
first_seen = LEAST(events.groups.first_seen, EXCLUDED.first_seen),
|
||||
last_seen = GREATEST(events.groups.last_seen, EXCLUDED.last_seen)
|
||||
RETURNING xmax = 0`,
|
||||
uuid.New(), e.OrgID, e.GroupKey.String, e.Source, e.Type, e.GroupFilter, e.ReceivedAt).Scan(&created)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("group event: %w", err)
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
|
||||
// Groups returns the groups no trigger covers, newest activity first. A group
|
||||
// stops being listed once a trigger exists for its source, type and filter.
|
||||
func (s *Store) Groups(ctx context.Context, orgID string, limit int) ([]Group, error) {
|
||||
var out []Group
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT g.id, g.source, g.event_type, g.filter, g.count, g.first_seen, g.last_seen
|
||||
FROM events.groups g
|
||||
WHERE g.org_id = $1 AND NOT EXISTS (
|
||||
SELECT 1 FROM events.triggers t
|
||||
WHERE t.org_id = g.org_id AND t.source = g.source AND t.event_type = g.event_type
|
||||
AND t.filter = g.filter)
|
||||
ORDER BY g.last_seen DESC LIMIT $2`, orgID, limit); err != nil {
|
||||
return nil, fmt.Errorf("list groups: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -26,6 +26,8 @@ type Source struct {
|
||||
TokenSHA256 string `db:"token_sha256"`
|
||||
// Cursor is where a listening source resumes.
|
||||
Cursor string `db:"cursor"`
|
||||
// KeyFields are the paths its events are grouped by; none means the default.
|
||||
KeyFields jsontext.Value `db:"key_fields"`
|
||||
}
|
||||
|
||||
// Event is one stored event.
|
||||
@@ -42,6 +44,12 @@ type Event struct {
|
||||
ReceivedAt time.Time `db:"received_at"`
|
||||
Matched bool `db:"matched"`
|
||||
DispatchedAt sql.NullTime `db:"dispatched_at"`
|
||||
// Own marks the echo of a delivery Archie sent, and OriginRunID is the run
|
||||
// that sent it. An own event has no group.
|
||||
Own bool `db:"own"`
|
||||
OriginRunID uuid.NullUUID `db:"origin_run_id"`
|
||||
GroupKey sql.NullString `db:"group_key"`
|
||||
GroupFilter jsontext.Value `db:"group_filter"`
|
||||
}
|
||||
|
||||
// Trigger starts a workflow for events of one type from one source whose
|
||||
@@ -73,7 +81,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} }
|
||||
func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) {
|
||||
var src Source
|
||||
err := s.db.GetContext(ctx, &src, `
|
||||
SELECT org_id, name, connector, token_sha256 FROM events.sources
|
||||
SELECT org_id, name, connector, token_sha256, key_fields FROM events.sources
|
||||
WHERE org_id = $1 AND name = $2`, orgID, name)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Source{}, ErrNotFound
|
||||
@@ -88,7 +96,7 @@ func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error)
|
||||
func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) {
|
||||
var out []Source
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT org_id, name, connector, token_sha256, cursor FROM events.sources
|
||||
SELECT org_id, name, connector, token_sha256, cursor, key_fields FROM events.sources
|
||||
WHERE org_id = $1 ORDER BY name`, orgID); err != nil {
|
||||
return nil, fmt.Errorf("list sources: %w", err)
|
||||
}
|
||||
@@ -121,8 +129,10 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error {
|
||||
// event id. It reports whether the event was new.
|
||||
func (s *Store) Insert(ctx context.Context, e Event) (bool, error) {
|
||||
res, err := s.db.NamedExecContext(ctx, `
|
||||
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at)
|
||||
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at)
|
||||
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at,
|
||||
own, origin_run_id, group_key, group_filter)
|
||||
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at,
|
||||
:own, :origin_run_id, :group_key, :group_filter)
|
||||
ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("store event: %w", err)
|
||||
@@ -147,7 +157,7 @@ func (s *Store) Dispatch(ctx context.Context, fn func(tx *sqlx.Tx, e Event) (mat
|
||||
var e Event
|
||||
err = tx.GetContext(ctx, &e, `
|
||||
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
|
||||
received_at, matched, dispatched_at
|
||||
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
|
||||
FROM events.events WHERE dispatched_at IS NULL
|
||||
ORDER BY received_at FOR UPDATE SKIP LOCKED LIMIT 1`)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
@@ -252,7 +262,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Event, e
|
||||
var out []Event
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT id, org_id, source, type, source_event_id, link_key, linked_to, fields, occurred_at,
|
||||
received_at, matched, dispatched_at
|
||||
received_at, matched, dispatched_at, own, origin_run_id, group_key, group_filter
|
||||
FROM events.events WHERE org_id = $1 ORDER BY received_at DESC LIMIT $2`, orgID, limit); err != nil {
|
||||
return nil, fmt.Errorf("list events: %w", err)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ function signOut() {
|
||||
<nav v-if="signedIn" class="flex flex-1 items-center gap-3 text-label">
|
||||
<RouterLink to="/composer" active-class="text-foreground" class="text-muted-foreground">Composer</RouterLink>
|
||||
<RouterLink to="/runs" active-class="text-foreground" class="text-muted-foreground">Runs</RouterLink>
|
||||
<RouterLink to="/groups" active-class="text-foreground" class="text-muted-foreground">Unmatched</RouterLink>
|
||||
<RouterLink to="/classes" active-class="text-foreground" class="text-muted-foreground">Classes</RouterLink>
|
||||
<RouterLink to="/people" active-class="text-foreground" class="text-muted-foreground">People</RouterLink>
|
||||
<RouterLink to="/policies" active-class="text-foreground" class="text-muted-foreground">Policies</RouterLink>
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
<script setup lang="ts">
|
||||
import { onBeforeUnmount, onMounted, ref } from "vue";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { ReadError } from "@/components/ui/read-error";
|
||||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
|
||||
import { Timestamp } from "@/components/ui/timestamp";
|
||||
import { listGroups, saveTrigger, Unauthorized, type EventGroup } from "./api";
|
||||
|
||||
const emit = defineEmits<{ unauthorized: [] }>();
|
||||
|
||||
const groups = ref<EventGroup[]>([]);
|
||||
const workflows = ref<Record<string, string>>({});
|
||||
const loading = ref(true);
|
||||
const creating = ref<string | null>(null);
|
||||
const error = ref("");
|
||||
const actionError = ref("");
|
||||
const controller = new AbortController();
|
||||
|
||||
async function load() {
|
||||
loading.value = true;
|
||||
error.value = "";
|
||||
try {
|
||||
groups.value = await listGroups(controller.signal);
|
||||
} catch (err) {
|
||||
fail(err, (message) => (error.value = message));
|
||||
} finally {
|
||||
loading.value = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** A trigger name unique to the group, so creating never replaces another trigger. */
|
||||
function triggerName(g: EventGroup): string {
|
||||
const base = `${g.proposed.source}-${g.proposed.event_type}`.toLowerCase().replace(/[^a-z0-9]+/g, "-");
|
||||
return `${base.slice(0, 50)}-${g.id.slice(0, 8)}`.replace(/^[^a-z]+/, "g");
|
||||
}
|
||||
|
||||
async function create(g: EventGroup) {
|
||||
const workflow = (workflows.value[g.id] ?? "").trim();
|
||||
if (!workflow) return;
|
||||
creating.value = g.id;
|
||||
actionError.value = "";
|
||||
try {
|
||||
await saveTrigger(triggerName(g), { ...g.proposed, workflow });
|
||||
groups.value = await listGroups(controller.signal);
|
||||
} catch (err) {
|
||||
fail(err, (message) => (actionError.value = message));
|
||||
} finally {
|
||||
creating.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
function fail(err: unknown, set: (message: string) => void) {
|
||||
if (controller.signal.aborted) return;
|
||||
if (err instanceof Unauthorized) emit("unauthorized");
|
||||
set(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
|
||||
const filterText = (filter: Record<string, string>) =>
|
||||
Object.entries(filter)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(" ");
|
||||
|
||||
onMounted(load);
|
||||
onBeforeUnmount(() => controller.abort());
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div>
|
||||
<Spinner v-if="loading" />
|
||||
<ReadError v-else-if="error" :message="error" retry-label="Retry" @retry="load" />
|
||||
<Empty v-else-if="groups.length === 0">
|
||||
<EmptyHeader><EmptyTitle>No unmatched events</EmptyTitle></EmptyHeader>
|
||||
</Empty>
|
||||
<template v-else>
|
||||
<p v-if="actionError" class="mb-2 text-caption text-danger">{{ actionError }}</p>
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Trigger</TableHead>
|
||||
<TableHead>Events</TableHead>
|
||||
<TableHead>Last seen</TableHead>
|
||||
<TableHead>Workflow</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
<TableRow v-for="g in groups" :key="g.id">
|
||||
<TableCell>
|
||||
<div>{{ g.proposed.source }} {{ g.proposed.event_type }}</div>
|
||||
<div class="font-mono text-caption text-muted-foreground">{{ filterText(g.proposed.filter) }}</div>
|
||||
</TableCell>
|
||||
<TableCell>{{ g.count }}</TableCell>
|
||||
<TableCell><Timestamp :at="g.last_seen" /></TableCell>
|
||||
<TableCell>
|
||||
<form class="flex gap-2" @submit.prevent="create(g)">
|
||||
<Input v-model="workflows[g.id]" aria-label="Workflow" />
|
||||
<Button type="submit" size="xs" :disabled="creating === g.id || !(workflows[g.id] ?? '').trim()">
|
||||
Create
|
||||
</Button>
|
||||
</form>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableBody>
|
||||
</Table>
|
||||
</template>
|
||||
</div>
|
||||
</template>
|
||||
+15
-1
@@ -312,7 +312,21 @@ export async function resetClass(id: string): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
export const listRuns = (signal?: AbortSignal) => get<Run[]>("/api/runs", signal);
|
||||
/** Unmatched events that share a source, type and key fields, with the trigger they propose. */
|
||||
export interface EventGroup {
|
||||
id: string;
|
||||
count: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
proposed: { source: string; event_type: string; filter: Record<string, string>; workflow: string };
|
||||
}
|
||||
|
||||
export const listGroups = (signal?: AbortSignal) => get<EventGroup[]>("/api/groups", signal);
|
||||
|
||||
export const saveTrigger = (name: string, trigger: EventGroup["proposed"]) =>
|
||||
put(`/api/triggers/${encodeURIComponent(name)}`, trigger);
|
||||
|
||||
export const listRuns =(signal?: AbortSignal) => get<Run[]>("/api/runs", signal);
|
||||
export const runTrail = (id: string, signal?: AbortSignal) => get<Trail>(`/api/runs/${encodeURIComponent(id)}`, signal);
|
||||
|
||||
/**
|
||||
|
||||
@@ -6,6 +6,7 @@ export default createRouter({
|
||||
{ path: "/", redirect: "/composer" },
|
||||
{ path: "/composer", name: "composer", component: () => import("./ComposerView.vue") },
|
||||
{ path: "/runs", name: "runs", component: () => import("./RunsView.vue") },
|
||||
{ path: "/groups", name: "groups", component: () => import("./GroupsView.vue") },
|
||||
{ path: "/classes", name: "classes", component: () => import("./ClassesView.vue") },
|
||||
{ path: "/classes/:id", name: "class", component: () => import("./ClassView.vue"), props: true },
|
||||
{ path: "/people", name: "people", component: () => import("./PeopleView.vue") },
|
||||
|
||||
Reference in new issue
Block a user