diff --git a/go.mod b/go.mod index e733931ea..22cc1542d 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/pressly/goose/v3 v3.30.1 - github.com/samcharles93/ai-sdk v0.8.0 + github.com/samcharles93/ai-sdk v0.8.1 github.com/synadia-io/callout.go v0.2.3 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 diff --git a/go.sum b/go.sum index 44d2f50cf..0a786d0cf 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= -github.com/samcharles93/ai-sdk v0.8.0 h1:C+Ia7z7LpLtSf8OjSjhvswUddchFQ/bQp/rytnKhsyI= -github.com/samcharles93/ai-sdk v0.8.0/go.mod h1:NNkiUZ2qfezDHhmF8zHibKsPgpU8b5jphDQ0n2t1KL0= +github.com/samcharles93/ai-sdk v0.8.1 h1:9mxfaudUhQDqAEdmrOVX9yXcCWMTYvkNl6LRSICxP8k= +github.com/samcharles93/ai-sdk v0.8.1/go.mod h1:NNkiUZ2qfezDHhmF8zHibKsPgpU8b5jphDQ0n2t1KL0= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= diff --git a/internal/platform/pg/migrations/00032_event_bodies.sql b/internal/platform/pg/migrations/00032_event_bodies.sql new file mode 100644 index 000000000..940d2fda1 --- /dev/null +++ b/internal/platform/pg/migrations/00032_event_bodies.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE events.events ADD COLUMN body bytea; + +-- +goose Down +ALTER TABLE events.events DROP COLUMN body; diff --git a/modules/events/events.go b/modules/events/events.go index 1caf3108a..6f7130c18 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -120,7 +120,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusBadRequest, err.Error()) return } - stored, duplicates, err := m.save(r.Context(), src, parsed) + stored, duplicates, err := m.save(r.Context(), src, parsed, body) if err != nil { httpjson.Error(w, http.StatusServiceUnavailable, err.Error()) return @@ -128,15 +128,15 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) { httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates}) } -// save stores each event, recording at arrival whether it echoes a delivery -// Archie sent, so those never reach grouping. -func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event) (stored, duplicates int, err error) { +// save stores each event with the payload it arrived in, recording at arrival +// whether it echoes a delivery Archie sent, so those never reach grouping. +func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte) (stored, duplicates int, err error) { for _, e := range parsed { run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey) if err != nil { return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err) } - isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, own, run)) + isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run)) if err != nil { return stored, duplicates, err } @@ -211,7 +211,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) { err = fmt.Errorf("connector %q isn't running", src.Connector) } else { err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error { - if _, _, err := m.save(ctx, src, events); err != nil { + if _, _, err := m.save(ctx, src, events, received(events)); err != nil { return err } if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil { @@ -528,3 +528,13 @@ func newToken() (string, error) { } return hex.EncodeToString(b), nil } + +// received renders a batch a listening connector streamed, so the stored body +// is what reached the run rather than only the fields parsed from it. +func received(events []connectors.Event) []byte { + body, err := json.Marshal(events) + if err != nil { + return nil + } + return body +} diff --git a/modules/events/grouping.go b/modules/events/grouping.go index d8c5589e2..865cca7fe 100644 --- a/modules/events/grouping.go +++ b/modules/events/grouping.go @@ -57,10 +57,10 @@ func keyPaths(src store.Source) []string { // delivery Archie sent (own) is recorded with the run that sent it and gets no // group key, so it can never form or join a group. Any other event's key is // fixed here, once, from the source's key fields. -func newRow(orgID string, src store.Source, e connectors.Event, own bool, runID uuid.UUID) store.Event { +func newRow(orgID string, src store.Source, e connectors.Event, body []byte, own bool, runID uuid.UUID) store.Event { row := store.Event{ ID: uuid.New(), OrgID: orgID, Source: src.Name, Type: e.Type, SourceEventID: e.SourceEventID, - Fields: e.Fields, Own: own, + Fields: e.Fields, Body: body, Own: own, } row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != "" row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero() diff --git a/modules/events/grouping_test.go b/modules/events/grouping_test.go index 5cf192b71..f48e615f7 100644 --- a/modules/events/grouping_test.go +++ b/modules/events/grouping_test.go @@ -18,10 +18,10 @@ func TestNewRowGrouping(t *testing.T) { ev := func(fields string) connectors.Event { return connectors.Event{Type: "alert.firing", SourceEventID: fields, Fields: []byte(fields)} } - disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), false, uuid.Nil) - again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), false, uuid.Nil) - other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), false, uuid.Nil) - own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), true, run) + disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), nil, false, uuid.Nil) + again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), nil, false, uuid.Nil) + other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), nil, false, uuid.Nil) + own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), nil, true, run) if !disk.GroupKey.Valid || disk.GroupKey != again.GroupKey { t.Errorf("same key fields gave different keys: %q, %q", disk.GroupKey.String, again.GroupKey.String) diff --git a/modules/events/internal/store/store.go b/modules/events/internal/store/store.go index 8e0deda49..4ecfcdd24 100644 --- a/modules/events/internal/store/store.go +++ b/modules/events/internal/store/store.go @@ -40,10 +40,14 @@ type Event struct { LinkKey sql.NullString `db:"link_key"` LinkedTo *uuid.UUID `db:"linked_to"` Fields jsontext.Value `db:"fields"` - OccurredAt sql.NullTime `db:"occurred_at"` - ReceivedAt time.Time `db:"received_at"` - Matched bool `db:"matched"` - DispatchedAt sql.NullTime `db:"dispatched_at"` + // Body is the payload as it arrived: the webhook body, or the batch a + // listening connector streamed. It is what an audit reads when the + // parsed fields leave a question open. + Body []byte `db:"body"` + OccurredAt sql.NullTime `db:"occurred_at"` + ReceivedAt time.Time `db:"received_at"` + Matched bool `db:"matched"` + DispatchedAt sql.NullTime `db:"dispatched_at"` // Own marks the echo of a delivery Archie sent, and OriginRunID is the run // that sent it. An own event has no group. Own bool `db:"own"` @@ -129,9 +133,9 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error { // event id. It reports whether the event was new. func (s *Store) Insert(ctx context.Context, e Event) (bool, error) { res, err := s.db.NamedExecContext(ctx, ` - INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at, + INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, body, occurred_at, own, origin_run_id, group_key, group_filter) - VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at, + VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :body, :occurred_at, :own, :origin_run_id, :group_key, :group_filter) ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e) if err != nil {