feat(events,runs): an event keeps the payload it arrived in, and every model call records the request it sent via ai-sdk v0.8.1 (audit requirement)
This commit is contained in:
7 files changed
+40
-21
No files matched your search
@@ -94,7 +94,7 @@ require (
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/opencontainers/image-spec v1.1.1
|
||||
github.com/pressly/goose/v3 v3.30.1
|
||||
github.com/samcharles93/ai-sdk v0.8.0
|
||||
github.com/samcharles93/ai-sdk v0.8.1
|
||||
github.com/synadia-io/callout.go v0.2.3
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
|
||||
@@ -200,8 +200,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g=
|
||||
github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
|
||||
github.com/samcharles93/ai-sdk v0.8.0 h1:C+Ia7z7LpLtSf8OjSjhvswUddchFQ/bQp/rytnKhsyI=
|
||||
github.com/samcharles93/ai-sdk v0.8.0/go.mod h1:NNkiUZ2qfezDHhmF8zHibKsPgpU8b5jphDQ0n2t1KL0=
|
||||
github.com/samcharles93/ai-sdk v0.8.1 h1:9mxfaudUhQDqAEdmrOVX9yXcCWMTYvkNl6LRSICxP8k=
|
||||
github.com/samcharles93/ai-sdk v0.8.1/go.mod h1:NNkiUZ2qfezDHhmF8zHibKsPgpU8b5jphDQ0n2t1KL0=
|
||||
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
|
||||
github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE events.events ADD COLUMN body bytea;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE events.events DROP COLUMN body;
|
||||
@@ -120,7 +120,7 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
stored, duplicates, err := m.save(r.Context(), src, parsed)
|
||||
stored, duplicates, err := m.save(r.Context(), src, parsed, body)
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusServiceUnavailable, err.Error())
|
||||
return
|
||||
@@ -128,15 +128,15 @@ func (m *Module) intake(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Write(w, http.StatusAccepted, map[string]int{"stored": stored, "duplicates": duplicates})
|
||||
}
|
||||
|
||||
// save stores each event, recording at arrival whether it echoes a delivery
|
||||
// Archie sent, so those never reach grouping.
|
||||
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event) (stored, duplicates int, err error) {
|
||||
// save stores each event with the payload it arrived in, recording at arrival
|
||||
// whether it echoes a delivery Archie sent, so those never reach grouping.
|
||||
func (m *Module) save(ctx context.Context, src store.Source, parsed []connectors.Event, body []byte) (stored, duplicates int, err error) {
|
||||
for _, e := range parsed {
|
||||
run, own, err := m.deliveries.SentBy(ctx, src.Connector, e.SourceEventID, e.LinkKey)
|
||||
if err != nil {
|
||||
return stored, duplicates, fmt.Errorf("check delivery provenance: %w", err)
|
||||
}
|
||||
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, own, run))
|
||||
isNew, err := m.store.Insert(ctx, newRow(m.orgID, src, e, body, own, run))
|
||||
if err != nil {
|
||||
return stored, duplicates, err
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func (m *Module) listen(ctx context.Context, src store.Source) {
|
||||
err = fmt.Errorf("connector %q isn't running", src.Connector)
|
||||
} else {
|
||||
err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error {
|
||||
if _, _, err := m.save(ctx, src, events); err != nil {
|
||||
if _, _, err := m.save(ctx, src, events, received(events)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil {
|
||||
@@ -528,3 +528,13 @@ func newToken() (string, error) {
|
||||
}
|
||||
return hex.EncodeToString(b), nil
|
||||
}
|
||||
|
||||
// received renders a batch a listening connector streamed, so the stored body
|
||||
// is what reached the run rather than only the fields parsed from it.
|
||||
func received(events []connectors.Event) []byte {
|
||||
body, err := json.Marshal(events)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return body
|
||||
}
|
||||
@@ -57,10 +57,10 @@ func keyPaths(src store.Source) []string {
|
||||
// delivery Archie sent (own) is recorded with the run that sent it and gets no
|
||||
// group key, so it can never form or join a group. Any other event's key is
|
||||
// fixed here, once, from the source's key fields.
|
||||
func newRow(orgID string, src store.Source, e connectors.Event, own bool, runID uuid.UUID) store.Event {
|
||||
func newRow(orgID string, src store.Source, e connectors.Event, body []byte, own bool, runID uuid.UUID) store.Event {
|
||||
row := store.Event{
|
||||
ID: uuid.New(), OrgID: orgID, Source: src.Name, Type: e.Type, SourceEventID: e.SourceEventID,
|
||||
Fields: e.Fields, Own: own,
|
||||
Fields: e.Fields, Body: body, Own: own,
|
||||
}
|
||||
row.LinkKey.String, row.LinkKey.Valid = e.LinkKey, e.LinkKey != ""
|
||||
row.OccurredAt.Time, row.OccurredAt.Valid = e.OccurredAt, !e.OccurredAt.IsZero()
|
||||
|
||||
@@ -18,10 +18,10 @@ func TestNewRowGrouping(t *testing.T) {
|
||||
ev := func(fields string) connectors.Event {
|
||||
return connectors.Event{Type: "alert.firing", SourceEventID: fields, Fields: []byte(fields)}
|
||||
}
|
||||
disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), false, uuid.Nil)
|
||||
again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), false, uuid.Nil)
|
||||
other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), false, uuid.Nil)
|
||||
own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), true, run)
|
||||
disk := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":91}`), nil, false, uuid.Nil)
|
||||
again := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"},"value":97}`), nil, false, uuid.Nil)
|
||||
other := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"b"}}`), nil, false, uuid.Nil)
|
||||
own := newRow("o", src, ev(`{"alertname":"DiskPressure","labels":{"host":"a"}}`), nil, true, run)
|
||||
|
||||
if !disk.GroupKey.Valid || disk.GroupKey != again.GroupKey {
|
||||
t.Errorf("same key fields gave different keys: %q, %q", disk.GroupKey.String, again.GroupKey.String)
|
||||
|
||||
@@ -40,10 +40,14 @@ type Event struct {
|
||||
LinkKey sql.NullString `db:"link_key"`
|
||||
LinkedTo *uuid.UUID `db:"linked_to"`
|
||||
Fields jsontext.Value `db:"fields"`
|
||||
OccurredAt sql.NullTime `db:"occurred_at"`
|
||||
ReceivedAt time.Time `db:"received_at"`
|
||||
Matched bool `db:"matched"`
|
||||
DispatchedAt sql.NullTime `db:"dispatched_at"`
|
||||
// Body is the payload as it arrived: the webhook body, or the batch a
|
||||
// listening connector streamed. It is what an audit reads when the
|
||||
// parsed fields leave a question open.
|
||||
Body []byte `db:"body"`
|
||||
OccurredAt sql.NullTime `db:"occurred_at"`
|
||||
ReceivedAt time.Time `db:"received_at"`
|
||||
Matched bool `db:"matched"`
|
||||
DispatchedAt sql.NullTime `db:"dispatched_at"`
|
||||
// Own marks the echo of a delivery Archie sent, and OriginRunID is the run
|
||||
// that sent it. An own event has no group.
|
||||
Own bool `db:"own"`
|
||||
@@ -129,9 +133,9 @@ func (s *Store) UpsertSource(ctx context.Context, src Source) error {
|
||||
// event id. It reports whether the event was new.
|
||||
func (s *Store) Insert(ctx context.Context, e Event) (bool, error) {
|
||||
res, err := s.db.NamedExecContext(ctx, `
|
||||
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, occurred_at,
|
||||
INSERT INTO events.events (id, org_id, source, type, source_event_id, link_key, fields, body, occurred_at,
|
||||
own, origin_run_id, group_key, group_filter)
|
||||
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :occurred_at,
|
||||
VALUES (:id, :org_id, :source, :type, :source_event_id, :link_key, :fields, :body, :occurred_at,
|
||||
:own, :origin_run_id, :group_key, :group_filter)
|
||||
ON CONFLICT (org_id, source, source_event_id) DO NOTHING`, e)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user