From 79b352dc444be3a198cbc30fb12ca6f49f462149 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 24 Sep 2026 14:21:20 +1000 Subject: [PATCH] feat(eventtype): infer event types from captures and gate dispatch on identification --- internal/domain/eventtype/eventtype.go | 213 ++++++++++++ internal/domain/eventtype/eventtype_test.go | 311 ++++++++++++++++++ internal/domain/eventtype/propose.go | 109 ++++++ internal/domain/eventtype/signature.go | 204 ++++++++++++ .../domain/storecontract/storecontract.go | 19 ++ internal/infrastructure/postgres/edastore.go | 14 +- .../postgres/edastore_eventtype_test.go | 152 +++++++++ .../postgres/edastore_eventtypes.go | 152 +++++++++ .../infrastructure/postgres/edastore_test.go | 4 + .../postgres/migrations/0007_event_types.sql | 29 ++ .../postgres/postgresdb/edastore.sql.go | 165 +++++++++- .../postgres/postgresdb/models.go | 11 + .../postgres/queries/edastore.sql | 37 ++- 13 files changed, 1409 insertions(+), 11 deletions(-) create mode 100644 internal/domain/eventtype/eventtype.go create mode 100644 internal/domain/eventtype/eventtype_test.go create mode 100644 internal/domain/eventtype/propose.go create mode 100644 internal/domain/eventtype/signature.go create mode 100644 internal/infrastructure/postgres/edastore_eventtype_test.go create mode 100644 internal/infrastructure/postgres/edastore_eventtypes.go create mode 100644 internal/infrastructure/postgres/migrations/0007_event_types.sql diff --git a/internal/domain/eventtype/eventtype.go b/internal/domain/eventtype/eventtype.go new file mode 100644 index 000000000..8d8c500e3 --- /dev/null +++ b/internal/domain/eventtype/eventtype.go @@ -0,0 +1,213 @@ +// Package eventtype owns event types: named kinds of event from one source, +// each defined by a match rule over headers and payload. It infers a +// structural signature per capture, groups a source's captures into proposed +// types, evaluates rules, and refuses two types on one source whose rules can +// match the same event. An event that matches no type is unidentified and is +// never dispatched. See docs/prds/event-automation.md, "Event types". +package eventtype + +import ( + "errors" + "fmt" + "maps" + "slices" + "strings" + "time" +) + +var ( + // ErrInvalid marks an event type that cannot be saved as written. + ErrInvalid = errors.New("eventtype: invalid event type") + // ErrOverlap marks an event type whose rule can match an event another + // type on the same source also matches. + ErrOverlap = errors.New("eventtype: rule overlaps another event type on the source") +) + +// Operator is a payload condition's test. +type Operator string + +const ( + OpEquals Operator = "equals" + OpPresent Operator = "present" +) + +// HeaderCondition holds when the named header (case-insensitive) has Value. +type HeaderCondition struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// PayloadCondition holds when Path is present or, for OpEquals, when a value +// at Path renders as Value ("7", "true", "null", or the string itself). +type PayloadCondition struct { + Path string `json:"path"` + Op Operator `json:"op"` + Value string `json:"value,omitempty"` +} + +// Rule is a conjunction: every condition must hold. An empty rule matches +// every event from its source. +type Rule struct { + Headers []HeaderCondition `json:"headers"` + Payload []PayloadCondition `json:"payload"` +} + +// EventType is a named kind of event from one source. Schema is the inferred +// structure (path to type) of the payload it was created from. +type EventType struct { + ID string `json:"id"` + Source string `json:"source"` + Name string `json:"name"` + Rule Rule `json:"rule"` + Schema map[string]ValueType `json:"schema"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Validate refuses an event type that has no name or source or carries a +// malformed condition. +func (t EventType) Validate() error { + if strings.TrimSpace(t.Name) == "" { + return fmt.Errorf("%w: name is required", ErrInvalid) + } + if strings.TrimSpace(t.Source) == "" { + return fmt.Errorf("%w: source is required", ErrInvalid) + } + for _, h := range t.Rule.Headers { + if strings.TrimSpace(h.Name) == "" { + return fmt.Errorf("%w: header condition has no name", ErrInvalid) + } + } + for _, p := range t.Rule.Payload { + if strings.TrimSpace(p.Path) == "" { + return fmt.Errorf("%w: payload condition has no path", ErrInvalid) + } + if p.Op != OpEquals && p.Op != OpPresent { + return fmt.Errorf("%w: unknown operator %q", ErrInvalid, p.Op) + } + } + return nil +} + +// Matches reports whether every condition of r holds for s. +func (r Rule) Matches(s Sample) bool { + for _, h := range r.Headers { + got, ok := headerLookup(s.Headers, h.Name) + if !ok || headerValue(h.Name, got) != headerValue(h.Name, h.Value) { + return false + } + } + if len(r.Payload) == 0 { + return true + } + root, ok := decode(s.Body) + if !ok { + return false + } + for _, p := range r.Payload { + if !payloadHolds(root, p) { + return false + } + } + return true +} + +func headerLookup(headers map[string]string, name string) (string, bool) { + for k, v := range headers { + if strings.EqualFold(k, name) { + return v, true + } + } + return "", false +} + +func payloadHolds(root any, p PayloadCondition) bool { + values := lookup(root, p.Path) + if p.Op == OpPresent { + return len(values) > 0 + } + for _, v := range values { + if text, ok := scalarText(v); ok && text == p.Value { + return true + } + } + return false +} + +// Overlaps reports whether some event could satisfy both rules. Two rules are +// disjoint only when they demand different values of one header, or different +// values at one payload path outside any array; anything else is treated as +// overlapping, which errs towards refusing a save rather than towards an event +// two types both claim. +func Overlaps(a, b Rule) bool { + for _, ha := range a.Headers { + for _, hb := range b.Headers { + if strings.EqualFold(ha.Name, hb.Name) && headerValue(ha.Name, ha.Value) != headerValue(hb.Name, hb.Value) { + return false + } + } + } + for _, pa := range a.Payload { + for _, pb := range b.Payload { + if pa.Op == OpEquals && pb.Op == OpEquals && pa.Path == pb.Path && + !strings.Contains(pa.Path, "[]") && pa.Value != pb.Value { + return false + } + } + } + return true +} + +// CheckOverlap refuses candidate when its rule overlaps another type on the +// same source. A type is not compared with itself, so an update passes. +func CheckOverlap(existing []EventType, candidate EventType) error { + for _, t := range existing { + if t.Source != candidate.Source || (candidate.ID != "" && t.ID == candidate.ID) { + continue + } + if Overlaps(t.Rule, candidate.Rule) { + return fmt.Errorf("%w: %q", ErrOverlap, t.Name) + } + } + return nil +} + +// Identify returns the one type on source whose rule matches s. No match, or +// more than one (which overlap refusal prevents), is unidentified. +func Identify(types []EventType, source string, s Sample) (EventType, bool) { + var found []EventType + for _, t := range types { + if t.Source == source && t.Rule.Matches(s) { + found = append(found, t) + } + } + if len(found) != 1 { + return EventType{}, false + } + return found[0], true +} + +// FromExample builds an event type from a pasted payload: its schema is the +// example's structure, and its rule is the example's discriminator headers or, +// when it has none, the presence of each top-level payload key. +func FromExample(source, name string, s Sample) EventType { + sig := Sign(s) + t := EventType{Source: source, Name: name, Schema: sig.Paths} + t.Rule.Headers = headerConditions(sig.Headers) + if len(t.Rule.Headers) == 0 { + for _, path := range slices.Sorted(maps.Keys(sig.Paths)) { + if !strings.ContainsAny(path, ".[") { + t.Rule.Payload = append(t.Rule.Payload, PayloadCondition{Path: path, Op: OpPresent}) + } + } + } + return t +} + +func headerConditions(headers map[string]string) []HeaderCondition { + var out []HeaderCondition + for _, name := range slices.Sorted(maps.Keys(headers)) { + out = append(out, HeaderCondition{Name: name, Value: headers[name]}) + } + return out +} diff --git a/internal/domain/eventtype/eventtype_test.go b/internal/domain/eventtype/eventtype_test.go new file mode 100644 index 000000000..0c21b7447 --- /dev/null +++ b/internal/domain/eventtype/eventtype_test.go @@ -0,0 +1,311 @@ +package eventtype + +import ( + "errors" + "reflect" + "testing" +) + +func sample(headers map[string]string, body string) Sample { + return Sample{Headers: headers, Body: []byte(body)} +} + +func TestParseHeaders(t *testing.T) { + tests := []struct { + name string + raw string + want map[string]string + }{ + {"http header json", `{"X-Github-Event":["push"],"Content-Type":["application/json"]}`, map[string]string{"x-github-event": "push", "content-type": "application/json"}}, + {"empty", ``, map[string]string{}}, + {"not json", `nope`, map[string]string{}}, + {"empty value list", `{"X-A":[]}`, map[string]string{}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ParseHeaders(tt.raw); !reflect.DeepEqual(got, tt.want) { + t.Fatalf("ParseHeaders = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSign(t *testing.T) { + tests := []struct { + name string + in Sample + wantPaths map[string]ValueType + wantHeaders map[string]string + }{ + { + name: "paths with types and discriminator headers", + in: sample(map[string]string{ + "x-github-event": "pull_request", + "content-type": "application/json; charset=utf-8", + "user-agent": "GitHub-Hookshot", + }, `{"action":"opened","number":7,"draft":false,"pull_request":{"labels":[{"name":"x"}],"body":null},"tags":[]}`), + wantPaths: map[string]ValueType{ + "action": TypeString, + "number": TypeNumber, + "draft": TypeBool, + "pull_request": TypeObject, + "pull_request.labels": TypeArray, + "pull_request.labels[]": TypeObject, + "pull_request.labels[].name": TypeString, + "pull_request.body": TypeNull, + "tags": TypeArray, + }, + wantHeaders: map[string]string{"x-github-event": "pull_request", "content-type": "application/json"}, + }, + { + name: "non-json body has no paths", + in: sample(map[string]string{"x-kind-type": "blocked"}, `plain text`), + wantPaths: map[string]ValueType{}, + wantHeaders: map[string]string{"x-kind-type": "blocked"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Sign(tt.in) + if !reflect.DeepEqual(got.Paths, tt.wantPaths) { + t.Errorf("paths = %v, want %v", got.Paths, tt.wantPaths) + } + if !reflect.DeepEqual(got.Headers, tt.wantHeaders) { + t.Errorf("headers = %v, want %v", got.Headers, tt.wantHeaders) + } + }) + } +} + +func TestSignatureKeyIgnoresValuesButNotShape(t *testing.T) { + a := Sign(sample(nil, `{"a":1,"b":"x"}`)) + b := Sign(sample(nil, `{"b":"y","a":2}`)) + c := Sign(sample(nil, `{"a":"1","b":"x"}`)) + if a.Key() != b.Key() { + t.Errorf("same shape, different keys: %q vs %q", a.Key(), b.Key()) + } + if a.Key() == c.Key() { + t.Errorf("different types share key %q", a.Key()) + } +} + +func TestRuleMatches(t *testing.T) { + ev := sample(map[string]string{"x-github-event": "pull_request", "content-type": "application/json; charset=utf-8"}, + `{"action":"opened","number":7,"merged":false,"labels":[{"name":"bug"},{"name":"ui"}]}`) + tests := []struct { + name string + rule Rule + want bool + }{ + {"empty rule matches", Rule{}, true}, + {"header equal, name case-insensitive", Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "pull_request"}}}, true}, + {"header differs", Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "push"}}}, false}, + {"header absent", Rule{Headers: []HeaderCondition{{Name: "X-Gitea-Event", Value: "push"}}}, false}, + {"content type compared by media type", Rule{Headers: []HeaderCondition{{Name: "Content-Type", Value: "application/json"}}}, true}, + {"path equals string", Rule{Payload: []PayloadCondition{{Path: "action", Op: OpEquals, Value: "opened"}}}, true}, + {"path equals other string", Rule{Payload: []PayloadCondition{{Path: "action", Op: OpEquals, Value: "closed"}}}, false}, + {"path equals number", Rule{Payload: []PayloadCondition{{Path: "number", Op: OpEquals, Value: "7"}}}, true}, + {"path equals bool", Rule{Payload: []PayloadCondition{{Path: "merged", Op: OpEquals, Value: "false"}}}, true}, + {"path present", Rule{Payload: []PayloadCondition{{Path: "number", Op: OpPresent}}}, true}, + {"path missing", Rule{Payload: []PayloadCondition{{Path: "sender.login", Op: OpPresent}}}, false}, + {"array element equals", Rule{Payload: []PayloadCondition{{Path: "labels[].name", Op: OpEquals, Value: "ui"}}}, true}, + {"all conditions must hold", Rule{ + Headers: []HeaderCondition{{Name: "x-github-event", Value: "pull_request"}}, + Payload: []PayloadCondition{{Path: "action", Op: OpEquals, Value: "closed"}}, + }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.rule.Matches(ev); got != tt.want { + t.Fatalf("Matches = %v, want %v", got, tt.want) + } + }) + } +} + +func TestOverlaps(t *testing.T) { + gh := func(v string) HeaderCondition { return HeaderCondition{Name: "X-GitHub-Event", Value: v} } + eq := func(p, v string) PayloadCondition { return PayloadCondition{Path: p, Op: OpEquals, Value: v} } + present := func(p string) PayloadCondition { return PayloadCondition{Path: p, Op: OpPresent} } + tests := []struct { + name string + a, b Rule + want bool + }{ + {"different header values are disjoint", Rule{Headers: []HeaderCondition{gh("push")}}, Rule{Headers: []HeaderCondition{gh("pull_request")}}, false}, + {"header names compared case-insensitively", Rule{Headers: []HeaderCondition{gh("push")}}, Rule{Headers: []HeaderCondition{{Name: "x-github-event", Value: "issues"}}}, false}, + {"same header overlaps", Rule{Headers: []HeaderCondition{gh("push")}}, Rule{Headers: []HeaderCondition{gh("push")}}, true}, + {"empty rule overlaps everything", Rule{}, Rule{Headers: []HeaderCondition{gh("push")}}, true}, + {"different equals on a path are disjoint", Rule{Payload: []PayloadCondition{eq("action", "opened")}}, Rule{Payload: []PayloadCondition{eq("action", "closed")}}, false}, + {"equals and present on one path overlap", Rule{Payload: []PayloadCondition{eq("action", "opened")}}, Rule{Payload: []PayloadCondition{present("action")}}, true}, + {"unrelated conditions overlap", Rule{Payload: []PayloadCondition{present("a")}}, Rule{Payload: []PayloadCondition{present("b")}}, true}, + {"array paths never prove disjointness", Rule{Payload: []PayloadCondition{eq("l[].n", "x")}}, Rule{Payload: []PayloadCondition{eq("l[].n", "y")}}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Overlaps(tt.a, tt.b); got != tt.want { + t.Fatalf("Overlaps = %v, want %v", got, tt.want) + } + if got := Overlaps(tt.b, tt.a); got != tt.want { + t.Fatalf("Overlaps (swapped) = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEventTypeValidate(t *testing.T) { + ok := EventType{Source: "gh", Name: "push", Rule: Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "push"}}}} + tests := []struct { + name string + mutate func(*EventType) + valid bool + }{ + {"valid", func(*EventType) {}, true}, + {"missing name", func(e *EventType) { e.Name = " " }, false}, + {"missing source", func(e *EventType) { e.Source = "" }, false}, + {"header without name", func(e *EventType) { e.Rule.Headers = []HeaderCondition{{Value: "x"}} }, false}, + {"payload without path", func(e *EventType) { e.Rule.Payload = []PayloadCondition{{Op: OpPresent}} }, false}, + {"unknown operator", func(e *EventType) { e.Rule.Payload = []PayloadCondition{{Path: "a", Op: "like"}} }, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := ok + e.Rule = Rule{Headers: append([]HeaderCondition(nil), ok.Rule.Headers...)} + tt.mutate(&e) + err := e.Validate() + if (err == nil) != tt.valid { + t.Fatalf("Validate = %v, valid want %v", err, tt.valid) + } + if err != nil && !errors.Is(err, ErrInvalid) { + t.Fatalf("Validate error %v is not ErrInvalid", err) + } + }) + } +} + +func TestCheckOverlap(t *testing.T) { + push := EventType{ID: "1", Source: "gh", Name: "push", Rule: Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "push"}}}} + tests := []struct { + name string + candidate EventType + wantErr bool + }{ + {"disjoint on same source", EventType{Source: "gh", Name: "pr", Rule: Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "pull_request"}}}}, false}, + {"overlapping on same source", EventType{Source: "gh", Name: "any", Rule: Rule{}}, true}, + {"overlapping on another source", EventType{Source: "fw", Name: "any", Rule: Rule{}}, false}, + {"an update does not overlap itself", EventType{ID: "1", Source: "gh", Name: "push2", Rule: Rule{}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CheckOverlap([]EventType{push}, tt.candidate) + if (err != nil) != tt.wantErr { + t.Fatalf("CheckOverlap = %v, wantErr %v", err, tt.wantErr) + } + if err != nil && !errors.Is(err, ErrOverlap) { + t.Fatalf("error %v is not ErrOverlap", err) + } + }) + } +} + +func TestIdentify(t *testing.T) { + types := []EventType{ + {ID: "push", Source: "gh", Name: "push", Rule: Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "push"}}}}, + {ID: "pr", Source: "gh", Name: "pr", Rule: Rule{Headers: []HeaderCondition{{Name: "X-GitHub-Event", Value: "pull_request"}}}}, + {ID: "fw", Source: "fw", Name: "any", Rule: Rule{}}, + } + tests := []struct { + name string + source string + in Sample + want string + }{ + {"matches its type", "gh", sample(map[string]string{"x-github-event": "push"}, `{}`), "push"}, + {"no type matches", "gh", sample(map[string]string{"x-github-event": "issues"}, `{}`), ""}, + {"types from other sources are ignored", "other", sample(nil, `{}`), ""}, + {"catch-all on its source", "fw", sample(nil, `{"x":1}`), "fw"}, + {"two matches is unidentified", "gh", sample(map[string]string{"x-github-event": "push"}, `{}`), ""}, + } + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := types + if i == len(tests)-1 { + ts = append(ts, EventType{ID: "dup", Source: "gh", Name: "dup", Rule: Rule{}}) + } + got, ok := Identify(ts, tt.source, tt.in) + if got.ID != tt.want || ok != (tt.want != "") { + t.Fatalf("Identify = %q,%v want %q", got.ID, ok, tt.want) + } + }) + } +} + +func TestFromExample(t *testing.T) { + tests := []struct { + name string + in Sample + wantRule Rule + }{ + { + name: "discriminator headers become the rule", + in: sample(map[string]string{"X-GitHub-Event": "push", "User-Agent": "x"}, `{"ref":"main"}`), + wantRule: Rule{Headers: []HeaderCondition{{Name: "x-github-event", Value: "push"}}}, + }, + { + name: "without discriminators the top-level paths must be present", + in: sample(nil, `{"src":"1.2.3.4","action":{"k":1}}`), + wantRule: Rule{Payload: []PayloadCondition{{Path: "action", Op: OpPresent}, {Path: "src", Op: OpPresent}}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := FromExample("s", "n", tt.in) + if !reflect.DeepEqual(got.Rule, tt.wantRule) { + t.Fatalf("rule = %+v, want %+v", got.Rule, tt.wantRule) + } + if !reflect.DeepEqual(got.Schema, Sign(tt.in).Paths) { + t.Fatalf("schema = %v, want the example's paths", got.Schema) + } + if !got.Rule.Matches(tt.in) { + t.Fatal("a type made from an example must match that example") + } + }) + } +} + +func TestPropose(t *testing.T) { + gh := func(id, ev, body string) Capture { + return Capture{ID: id, Source: "gh", Sample: sample(map[string]string{"x-github-event": ev}, body)} + } + captures := []Capture{ + gh("c1", "push", `{"ref":"a"}`), + gh("c2", "pull_request", `{"action":"opened"}`), + gh("c3", "push", `{"ref":"b"}`), + {ID: "f1", Source: "fw", Sample: sample(nil, `{"src":"1.1.1.1","port":22}`)}, + {ID: "f2", Source: "fw", Sample: sample(nil, `{"src":"1.1.1.1","port":22,"extra":true}`)}, + } + got := Propose(captures) + type summary struct { + source, sample string + count int + rule Rule + } + var sums []summary + for _, p := range got { + sums = append(sums, summary{p.Source, p.SampleID, p.Count, p.Rule}) + } + want := []summary{ + {"fw", "f1", 1, Rule{}}, + {"fw", "f2", 1, Rule{Payload: []PayloadCondition{{Path: "extra", Op: OpPresent}}}}, + {"gh", "c1", 2, Rule{Headers: []HeaderCondition{{Name: "x-github-event", Value: "push"}}}}, + {"gh", "c2", 1, Rule{Headers: []HeaderCondition{{Name: "x-github-event", Value: "pull_request"}}}}, + } + if !reflect.DeepEqual(sums, want) { + t.Fatalf("Propose =\n%+v\nwant\n%+v", sums, want) + } + for _, p := range got { + if len(p.Schema) == 0 { + t.Errorf("proposal %s has no schema", p.SampleID) + } + } +} diff --git a/internal/domain/eventtype/propose.go b/internal/domain/eventtype/propose.go new file mode 100644 index 000000000..29dd32ec5 --- /dev/null +++ b/internal/domain/eventtype/propose.go @@ -0,0 +1,109 @@ +package eventtype + +import ( + "maps" + "slices" + "strings" + "time" +) + +// Capture is one stored event as proposal grouping reads it. +type Capture struct { + ID string + Source string + ReceivedAt time.Time + Sample Sample +} + +// Proposal is a group of one source's captures sharing a signature, offered +// to the operator as an event type to name. Rule is the set of +// discriminators that separates the group from its siblings on the source. +type Proposal struct { + Source string `json:"source"` + Key string `json:"key"` + Schema map[string]ValueType `json:"schema"` + Headers map[string]string `json:"headers"` + Count int `json:"count"` + SampleID string `json:"sample_id"` + Rule Rule `json:"rule"` +} + +// Propose groups captures by source and signature. Groups are ordered by +// source, then largest first, then first seen; the sample is the group's first +// capture in input order. +func Propose(captures []Capture) []Proposal { + var groups []*Proposal + sigs := map[*Proposal]Signature{} + index := map[string]*Proposal{} + for _, c := range captures { + sig := Sign(c.Sample) + key := sig.Key() + id := c.Source + "\x00" + key + if g, ok := index[id]; ok { + g.Count++ + continue + } + g := &Proposal{Source: c.Source, Key: key, Schema: sig.Paths, Headers: sig.Headers, Count: 1, SampleID: c.ID} + index[id] = g + sigs[g] = sig + groups = append(groups, g) + } + slices.SortStableFunc(groups, func(a, b *Proposal) int { + if c := strings.Compare(a.Source, b.Source); c != 0 { + return c + } + return b.Count - a.Count + }) + out := make([]Proposal, 0, len(groups)) + for _, g := range groups { + var siblings []Signature + for _, s := range groups { + if s != g && s.Source == g.Source { + siblings = append(siblings, sigs[s]) + } + } + g.Rule = separatingRule(sigs[g], siblings) + out = append(out, *g) + } + return out +} + +// separatingRule demands the group's discriminator headers, then, for each +// sibling those do not already rule out, the presence of the first path the +// group has and the sibling lacks. A group whose paths are a subset of a +// sibling's cannot be separated by presence alone; saving it as proposed is +// then refused as an overlap, and the operator edits the rule. +func separatingRule(g Signature, siblings []Signature) Rule { + rule := Rule{Headers: headerConditions(g.Headers)} + paths := slices.Sorted(maps.Keys(g.Paths)) + for _, s := range siblings { + if headersSeparate(g.Headers, s.Headers) || rulePathsSeparate(rule, s) { + continue + } + for _, path := range paths { + if _, ok := s.Paths[path]; !ok { + rule.Payload = append(rule.Payload, PayloadCondition{Path: path, Op: OpPresent}) + break + } + } + } + return rule +} + +func headersSeparate(g, s map[string]string) bool { + for name, value := range g { + if other, ok := s[name]; !ok || other != value { + return true + } + } + return false +} + +func rulePathsSeparate(rule Rule, s Signature) bool { + for _, p := range rule.Payload { + if _, ok := s.Paths[p.Path]; !ok { + return true + } + } + return false +} diff --git a/internal/domain/eventtype/signature.go b/internal/domain/eventtype/signature.go new file mode 100644 index 000000000..065844978 --- /dev/null +++ b/internal/domain/eventtype/signature.go @@ -0,0 +1,204 @@ +package eventtype + +import ( + "bytes" + "encoding/json" + "maps" + "mime" + "slices" + "strings" +) + +// ValueType is the JSON shape of one payload path. The names match +// mapping.FieldType so a mapping can be checked against a type's schema. +type ValueType string + +const ( + TypeString ValueType = "string" + TypeNumber ValueType = "number" + TypeBool ValueType = "bool" + TypeObject ValueType = "object" + TypeArray ValueType = "array" + TypeNull ValueType = "null" +) + +// Sample is one event as the rules see it: headers keyed by lower-case name +// (first value only) and the raw body. +type Sample struct { + Headers map[string]string + Body []byte +} + +// Signature is the structure of one event: every JSON path with its value +// type, plus the discriminator headers with their values. Array elements share +// one path segment, "[]", so two payloads differing only in list length have +// the same signature. +type Signature struct { + Paths map[string]ValueType `json:"paths"` + Headers map[string]string `json:"headers"` +} + +// ParseHeaders reads a capture's stored headers (an encoded http.Header) into +// the lower-case, first-value form a Sample carries. Anything unreadable is no +// headers, not an error: a capture's headers are evidence, not input. +func ParseHeaders(raw string) map[string]string { + out := map[string]string{} + var h map[string][]string + if json.Unmarshal([]byte(raw), &h) != nil { + return out + } + for name, values := range h { + if len(values) > 0 { + out[strings.ToLower(name)] = values[0] + } + } + return out +} + +// isDiscriminator reports whether a header names the kind of event rather than +// describing its transport: X-GitHub-Event, X-Gitea-Event-Type, Content-Type. +func isDiscriminator(name string) bool { + name = strings.ToLower(name) + return strings.HasSuffix(name, "-event") || strings.HasSuffix(name, "-type") +} + +// headerValue normalises a header value for comparison. Content-Type compares +// by media type, so a charset parameter does not split one kind of event in two. +func headerValue(name, value string) string { + value = strings.TrimSpace(value) + if strings.EqualFold(name, "content-type") { + if mt, _, err := mime.ParseMediaType(value); err == nil { + return mt + } + } + return value +} + +// Sign computes an event's structural signature. A body that is not JSON has +// no paths; its headers still discriminate it. +func Sign(s Sample) Signature { + sig := Signature{Paths: map[string]ValueType{}, Headers: map[string]string{}} + for name, value := range s.Headers { + if isDiscriminator(name) { + sig.Headers[strings.ToLower(name)] = headerValue(name, value) + } + } + if root, ok := decode(s.Body); ok { + collectPaths(root, "", sig.Paths) + } + return sig +} + +// Key is a stable string that is equal for two signatures exactly when their +// paths, path types and discriminator headers are equal. +func (s Signature) Key() string { + var b strings.Builder + for _, name := range slices.Sorted(maps.Keys(s.Headers)) { + b.WriteString("h:" + name + "=" + s.Headers[name] + "\n") + } + for _, path := range slices.Sorted(maps.Keys(s.Paths)) { + b.WriteString("p:" + path + "=" + string(s.Paths[path]) + "\n") + } + return b.String() +} + +func decode(body []byte) (any, bool) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, false + } + return v, true +} + +func valueType(v any) ValueType { + switch v.(type) { + case string: + return TypeString + case json.Number: + return TypeNumber + case bool: + return TypeBool + case map[string]any: + return TypeObject + case []any: + return TypeArray + default: + return TypeNull + } +} + +// collectPaths records every path below v. The root itself has no path. +func collectPaths(v any, prefix string, out map[string]ValueType) { + switch typed := v.(type) { + case map[string]any: + for key, child := range typed { + path := key + if prefix != "" { + path = prefix + "." + key + } + out[path] = valueType(child) + collectPaths(child, path, out) + } + case []any: + for _, child := range typed { + path := prefix + "[]" + out[path] = valueType(child) + collectPaths(child, path, out) + } + } +} + +// lookup returns every value at path; "[]" segments fan out over array +// elements, so a path inside a list yields one value per element that has it. +func lookup(root any, path string) []any { + current := []any{root} + for seg := range strings.SplitSeq(path, ".") { + name, arrays := seg, 0 + for strings.HasSuffix(name, "[]") { + name, arrays = strings.TrimSuffix(name, "[]"), arrays+1 + } + var next []any + for _, node := range current { + obj, ok := node.(map[string]any) + if !ok { + continue + } + if child, ok := obj[name]; ok { + next = append(next, child) + } + } + for range arrays { + var elems []any + for _, node := range next { + if arr, ok := node.([]any); ok { + elems = append(elems, arr...) + } + } + next = elems + } + current = next + } + return current +} + +// scalarText renders a scalar the way an operator types it into an equals +// condition; objects and arrays have no text and never equal anything. +func scalarText(v any) (string, bool) { + switch typed := v.(type) { + case string: + return typed, true + case json.Number: + return typed.String(), true + case bool: + if typed { + return "true", true + } + return "false", true + case nil: + return "null", true + default: + return "", false + } +} diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index 74628182b..483e89618 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -22,6 +22,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/eventtype" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/workflow/task" "github.com/samcharles93/archie-core/internal/events" @@ -177,6 +178,18 @@ type MappingStore interface { DeleteMapping(ctx context.Context, id string) error } +// EventTypeStore persists event types (docs/prds/event-automation.md, "Event +// types"). Insert and Update refuse a type whose rule overlaps another on the +// same source with eventtype.ErrOverlap, and a malformed one with +// eventtype.ErrInvalid. Update rewrites the name and rule; the schema is the +// one the type was created with. +type EventTypeStore interface { + InsertEventType(ctx context.Context, t eventtype.EventType) (string, error) + UpdateEventType(ctx context.Context, t eventtype.EventType) error + DeleteEventType(ctx context.Context, id string) error + ListEventTypes(ctx context.Context) ([]eventtype.EventType, error) +} + // BindingStore persists playbook bindings (t2db.4 Phase B): CRUD and the // draft -> pending_approval -> armed state machine. Split off from // TaskStore and MappingStore so the webui binding editor does not acquire @@ -271,6 +284,10 @@ type CapturedEvent struct { Headers string `json:"headers"` Body string `json:"body"` Authenticated bool `json:"authenticated"` + // EventType is the ID of the event type the capture was identified as + // when it arrived, or empty when it matched none. An unidentified capture + // is never dispatched. + EventType string `json:"event_type"` } // ConfigSnapshot is the running configuration as the dashboard renders it, @@ -370,6 +387,8 @@ var ( ErrAlreadyDispatched = errors.New("store: binding already dispatched for capture") // ErrMappingNotFound is returned when a mapping ID does not exist. ErrMappingNotFound = errors.New("store: mapping not found") + // ErrEventTypeNotFound is returned when an event type ID does not exist. + ErrEventTypeNotFound = errors.New("store: event type not found") // ErrResourceNotFound is returned when a control-plane resource kind has // no stored document. It is in-process only (not on the gRPC wire): the diff --git a/internal/infrastructure/postgres/edastore.go b/internal/infrastructure/postgres/edastore.go index 9c7cb2481..75aac0422 100644 --- a/internal/infrastructure/postgres/edastore.go +++ b/internal/infrastructure/postgres/edastore.go @@ -76,7 +76,8 @@ func newRecordID() string { return uuid.NewString() } // --- captures --- -// InsertCapture stores one inbound event verbatim. retention and maxEvents are +// InsertCapture stores one inbound event verbatim, tagged with the event type +// it is identified as on arrival (empty when unidentified). retention and maxEvents are // accepted for contract compatibility and applied as a prune after the insert, // exactly as the PocketBase store does. func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent, retention time.Duration, maxEvents int) (string, error) { @@ -84,6 +85,10 @@ func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent, if received.IsZero() { received = time.Now() } + eventType, err := s.identifyCapture(ctx, c) + if err != nil { + return "", err + } id := newRecordID() if err := s.q.InsertCapture(ctx, postgresdb.InsertCaptureParams{ ID: id, @@ -94,6 +99,7 @@ func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent, Body: c.Body, Authenticated: c.Authenticated, ReceivedAt: received.UTC(), + EventType: eventType, }); err != nil { return "", fmt.Errorf("edastore: insert capture: %w", err) } @@ -127,6 +133,7 @@ func captureValue(r postgresdb.Capture) storecontract.CapturedEvent { Headers: r.Headers, Body: r.Body, Authenticated: r.Authenticated, + EventType: r.EventType, } } @@ -143,8 +150,9 @@ func (s *EDA) ListCaptures(ctx context.Context, limit int) ([]storecontract.Capt return out, nil } -// ListUndispatchedCaptures returns recent captures for the given sources that -// no binding has dispatched yet. +// ListUndispatchedCaptures returns recent identified captures for the given +// sources that no binding has dispatched yet. An unidentified capture is +// never returned, so it never dispatches. func (s *EDA) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) { if len(sources) == 0 || limit <= 0 { return nil, nil diff --git a/internal/infrastructure/postgres/edastore_eventtype_test.go b/internal/infrastructure/postgres/edastore_eventtype_test.go new file mode 100644 index 000000000..2546ee821 --- /dev/null +++ b/internal/infrastructure/postgres/edastore_eventtype_test.go @@ -0,0 +1,152 @@ +package postgres + +import ( + "errors" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/eventtype" + "github.com/samcharles93/archie-core/internal/domain/storecontract" +) + +func githubType(name, event string) eventtype.EventType { + return eventtype.EventType{ + Source: "gh", Name: name, + Rule: eventtype.Rule{Headers: []eventtype.HeaderCondition{{Name: "X-GitHub-Event", Value: event}}}, + } +} + +func TestEventTypeSaveRefusals(t *testing.T) { + s := edaFor(t) + pushID, err := s.InsertEventType(t.Context(), githubType("push", "push")) + if err != nil { + t.Fatalf("InsertEventType(push) error = %v", err) + } + prID, err := s.InsertEventType(t.Context(), githubType("pr", "pull_request")) + if err != nil { + t.Fatalf("InsertEventType(pr) error = %v", err) + } + + tests := []struct { + name string + save func() error + wantErr error + }{ + {"insert overlapping rule", func() error { + _, err := s.InsertEventType(t.Context(), githubType("push-again", "push")) + return err + }, eventtype.ErrOverlap}, + {"insert catch-all beside typed", func() error { + _, err := s.InsertEventType(t.Context(), eventtype.EventType{Source: "gh", Name: "all"}) + return err + }, eventtype.ErrOverlap}, + {"insert duplicate name", func() error { + _, err := s.InsertEventType(t.Context(), githubType("push", "issues")) + return err + }, eventtype.ErrInvalid}, + {"insert without name", func() error { + _, err := s.InsertEventType(t.Context(), githubType("", "issues")) + return err + }, eventtype.ErrInvalid}, + {"update into overlap", func() error { + pr := githubType("pr", "push") + pr.ID = prID + return s.UpdateEventType(t.Context(), pr) + }, eventtype.ErrOverlap}, + {"update missing", func() error { + missing := githubType("x", "x") + missing.ID = "missing" + return s.UpdateEventType(t.Context(), missing) + }, storecontract.ErrEventTypeNotFound}, + {"delete missing", func() error { return s.DeleteEventType(t.Context(), "missing") }, storecontract.ErrEventTypeNotFound}, + {"other source is independent", func() error { + _, err := s.InsertEventType(t.Context(), eventtype.EventType{Source: "fw", Name: "all"}) + return err + }, nil}, + {"update keeps itself out of the overlap check", func() error { + push := githubType("push-renamed", "push") + push.ID = pushID + return s.UpdateEventType(t.Context(), push) + }, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.save() + if tt.wantErr == nil && err != nil { + t.Fatalf("error = %v, want nil", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + }) + } + + types, err := s.ListEventTypes(t.Context()) + if err != nil { + t.Fatalf("ListEventTypes() error = %v", err) + } + names := map[string]bool{} + for _, et := range types { + names[et.Source+"/"+et.Name] = true + } + for _, want := range []string{"gh/push-renamed", "gh/pr", "fw/all"} { + if !names[want] { + t.Errorf("ListEventTypes() lacks %s: %v", want, names) + } + } + if len(types) != 3 { + t.Errorf("ListEventTypes() = %d types, want 3", len(types)) + } +} + +// A type made from a pasted payload has that payload's schema, and a later +// matching capture is identified as it and becomes dispatchable; a capture +// matching no type stays unidentified and is never listed for dispatch. +func TestPastedTypeIdentifiesLaterCaptures(t *testing.T) { + s := edaFor(t) + pasted := eventtype.Sample{ + Headers: map[string]string{"X-GitHub-Event": "pull_request"}, + Body: []byte(`{"action":"opened","number":1}`), + } + id, err := s.InsertEventType(t.Context(), eventtype.FromExample("gh", "pull_request", pasted)) + if err != nil { + t.Fatalf("InsertEventType() error = %v", err) + } + types, err := s.ListEventTypes(t.Context()) + if err != nil || len(types) != 1 { + t.Fatalf("ListEventTypes() = %v (err %v)", types, err) + } + if got := types[0].Schema; got["action"] != eventtype.TypeString || got["number"] != eventtype.TypeNumber { + t.Fatalf("schema = %v, want the pasted payload's", got) + } + + for _, c := range []storecontract.CapturedEvent{ + {Source: "gh", Headers: `{"X-Github-Event":["pull_request"]}`, Body: `{"action":"closed","number":9}`}, + {Source: "gh", Headers: `{"X-Github-Event":["push"]}`, Body: `{"ref":"main"}`}, + } { + if _, err := s.InsertCapture(t.Context(), c, 0, 0); err != nil { + t.Fatalf("InsertCapture() error = %v", err) + } + } + all, err := s.ListCaptures(t.Context(), 10) + if err != nil { + t.Fatalf("ListCaptures() error = %v", err) + } + byBody := map[string]string{} + for _, c := range all { + byBody[c.Body] = c.EventType + } + if byBody[`{"action":"closed","number":9}`] != id { + t.Errorf("matching capture event type = %q, want %q", byBody[`{"action":"closed","number":9}`], id) + } + if byBody[`{"ref":"main"}`] != "" { + t.Errorf("unmatched capture event type = %q, want unidentified", byBody[`{"ref":"main"}`]) + } + + undispatched, err := s.ListUndispatchedCaptures(t.Context(), []string{"gh"}, 10) + if err != nil { + t.Fatalf("ListUndispatchedCaptures() error = %v", err) + } + if len(undispatched) != 1 || undispatched[0].EventType != id { + t.Fatalf("ListUndispatchedCaptures() = %+v, want only the identified capture", undispatched) + } +} diff --git a/internal/infrastructure/postgres/edastore_eventtypes.go b/internal/infrastructure/postgres/edastore_eventtypes.go new file mode 100644 index 000000000..caff14477 --- /dev/null +++ b/internal/infrastructure/postgres/edastore_eventtypes.go @@ -0,0 +1,152 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/samcharles93/archie-core/internal/domain/eventtype" + "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/infrastructure/postgres/postgresdb" +) + +var _ storecontract.EventTypeStore = (*EDA)(nil) + +func eventTypeValue(r postgresdb.EventType) (eventtype.EventType, error) { + t := eventtype.EventType{ID: r.ID, Source: r.Source, Name: r.Name, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt} + if err := json.Unmarshal([]byte(r.Rule), &t.Rule); err != nil { + return eventtype.EventType{}, fmt.Errorf("edastore: decode event type rule: %w", err) + } + if err := json.Unmarshal([]byte(r.Schema), &t.Schema); err != nil { + return eventtype.EventType{}, fmt.Errorf("edastore: decode event type schema: %w", err) + } + return t, nil +} + +func eventTypeValues(rows []postgresdb.EventType) ([]eventtype.EventType, error) { + out := make([]eventtype.EventType, 0, len(rows)) + for _, r := range rows { + t, err := eventTypeValue(r) + if err != nil { + return nil, err + } + out = append(out, t) + } + return out, nil +} + +// identifyCapture returns the ID of the one event type on the capture's +// source that matches it, or "" when it is unidentified. +func (s *EDA) identifyCapture(ctx context.Context, c storecontract.CapturedEvent) (string, error) { + rows, err := s.q.EventTypesForSource(ctx, c.Source) + if err != nil { + return "", fmt.Errorf("edastore: load event types: %w", err) + } + types, err := eventTypeValues(rows) + if err != nil { + return "", err + } + t, _ := eventtype.Identify(types, c.Source, eventtype.Sample{ + Headers: eventtype.ParseHeaders(c.Headers), + Body: []byte(c.Body), + }) + return t.ID, nil +} + +// saveEventType validates t, then, under a per-source lock, refuses it if its +// rule overlaps another type on the source and runs write. +func (s *EDA) saveEventType(ctx context.Context, t eventtype.EventType, write func(*postgresdb.Queries, string) error) error { + if err := t.Validate(); err != nil { + return err + } + rule, err := json.Marshal(t.Rule) + if err != nil { + return fmt.Errorf("edastore: encode event type rule: %w", err) + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + q := s.q.WithTx(tx) + if err := q.LockEventTypeSource(ctx, t.Source); err != nil { + return fmt.Errorf("edastore: lock event types: %w", err) + } + rows, err := q.EventTypesForSource(ctx, t.Source) + if err != nil { + return fmt.Errorf("edastore: load event types: %w", err) + } + existing, err := eventTypeValues(rows) + if err != nil { + return err + } + if err := eventtype.CheckOverlap(existing, t); err != nil { + return err + } + err = write(q, string(rule)) + if isUniqueViolation(err) { + return fmt.Errorf("%w: name %q is taken on source %q", eventtype.ErrInvalid, t.Name, t.Source) + } + return err + }) +} + +// InsertEventType stores a new event type. +func (s *EDA) InsertEventType(ctx context.Context, t eventtype.EventType) (string, error) { + t.ID = "" + schema, err := json.Marshal(t.Schema) + if err != nil { + return "", fmt.Errorf("edastore: encode event type schema: %w", err) + } + id := newRecordID() + err = s.saveEventType(ctx, t, func(q *postgresdb.Queries, rule string) error { + return q.InsertEventType(ctx, postgresdb.InsertEventTypeParams{ + ID: id, Source: t.Source, Name: t.Name, Rule: rule, Schema: string(schema), + }) + }) + if err != nil { + return "", err + } + return id, nil +} + +// UpdateEventType rewrites a type's name and rule. Its source and schema are +// the stored ones. +func (s *EDA) UpdateEventType(ctx context.Context, t eventtype.EventType) error { + stored, err := s.q.GetEventType(ctx, t.ID) + if errors.Is(err, pgx.ErrNoRows) { + return storecontract.ErrEventTypeNotFound + } + if err != nil { + return err + } + t.Source = stored.Source + return s.saveEventType(ctx, t, func(q *postgresdb.Queries, rule string) error { + n, err := q.UpdateEventType(ctx, postgresdb.UpdateEventTypeParams{ID: t.ID, Name: t.Name, Rule: rule}) + if err == nil && n == 0 { + return storecontract.ErrEventTypeNotFound + } + return err + }) +} + +// DeleteEventType removes a type. Captures already identified as it keep the +// ID they were stored with. +func (s *EDA) DeleteEventType(ctx context.Context, id string) error { + n, err := s.q.DeleteEventType(ctx, id) + if err != nil { + return err + } + if n == 0 { + return storecontract.ErrEventTypeNotFound + } + return nil +} + +// ListEventTypes returns every event type, by source then name. +func (s *EDA) ListEventTypes(ctx context.Context) ([]eventtype.EventType, error) { + rows, err := s.q.ListEventTypes(ctx) + if err != nil { + return nil, fmt.Errorf("edastore: list event types: %w", err) + } + return eventTypeValues(rows) +} diff --git a/internal/infrastructure/postgres/edastore_test.go b/internal/infrastructure/postgres/edastore_test.go index 71a934b14..c8baf1c76 100644 --- a/internal/infrastructure/postgres/edastore_test.go +++ b/internal/infrastructure/postgres/edastore_test.go @@ -9,6 +9,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/samcharles93/archie-core/internal/domain/binding" + "github.com/samcharles93/archie-core/internal/domain/eventtype" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/events" @@ -222,6 +223,9 @@ func TestBindingLifecycle(t *testing.T) { // out of the undispatched listing. func TestCaptureRoundTripAndUndispatched(t *testing.T) { s := edaFor(t) + if _, err := s.InsertEventType(t.Context(), eventtype.EventType{Source: "sentry", Name: "any"}); err != nil { + t.Fatalf("InsertEventType() error = %v", err) + } body := `{"action":"created"}` if _, err := s.InsertCapture(t.Context(), storecontract.CapturedEvent{ Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json", diff --git a/internal/infrastructure/postgres/migrations/0007_event_types.sql b/internal/infrastructure/postgres/migrations/0007_event_types.sql new file mode 100644 index 000000000..a7990c2f6 --- /dev/null +++ b/internal/infrastructure/postgres/migrations/0007_event_types.sql @@ -0,0 +1,29 @@ +-- Event types: named kinds of event from one source, each defined by a match +-- rule (docs/prds/event-automation.md, "Event types"). rule and schema are the +-- JSON encodings of eventtype.Rule and the inferred path-to-type schema; the +-- store evaluates them in Go, so they are opaque text here. +-- +-- captures.event_type records the type a capture was identified as when it +-- arrived. Empty means unidentified, and an unidentified capture is never +-- listed for dispatch. It is a plain text column, not a foreign key: deleting +-- a type must not rewrite what past captures were identified as. + +-- +goose Up + +CREATE TABLE event_types ( + id text PRIMARY KEY, + source text NOT NULL, + name text NOT NULL, + rule text NOT NULL, + schema text NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX idx_event_types_source_name ON event_types (source, name); + +ALTER TABLE captures ADD COLUMN event_type text NOT NULL DEFAULT ''; + +-- +goose Down + +ALTER TABLE captures DROP COLUMN event_type; +DROP TABLE event_types; diff --git a/internal/infrastructure/postgres/postgresdb/edastore.sql.go b/internal/infrastructure/postgres/postgresdb/edastore.sql.go index 9a2df6114..ab6fb99f7 100644 --- a/internal/infrastructure/postgres/postgresdb/edastore.sql.go +++ b/internal/infrastructure/postgres/postgresdb/edastore.sql.go @@ -80,6 +80,18 @@ func (q *Queries) DeleteCapturesOlderThan(ctx context.Context, receivedAt time.T return err } +const deleteEventType = `-- name: DeleteEventType :execrows +DELETE FROM event_types WHERE id = $1 +` + +func (q *Queries) DeleteEventType(ctx context.Context, id string) (int64, error) { + result, err := q.db.Exec(ctx, deleteEventType, id) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const deleteMapping = `-- name: DeleteMapping :execrows DELETE FROM mappings WHERE id = $1 ` @@ -101,6 +113,39 @@ func (q *Queries) DeletePlaybookDispatches(ctx context.Context, playbookID strin return err } +const eventTypesForSource = `-- name: EventTypesForSource :many +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types WHERE source = $1 ORDER BY name +` + +func (q *Queries) EventTypesForSource(ctx context.Context, source string) ([]EventType, error) { + rows, err := q.db.Query(ctx, eventTypesForSource, source) + if err != nil { + return nil, err + } + defer rows.Close() + var items []EventType + for rows.Next() { + var i EventType + if err := rows.Scan( + &i.ID, + &i.Source, + &i.Name, + &i.Rule, + &i.Schema, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getBinding = `-- name: GetBinding :one SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at FROM bindings WHERE id = $1 @@ -126,6 +171,26 @@ func (q *Queries) GetBinding(ctx context.Context, id string) (Binding, error) { return i, err } +const getEventType = `-- name: GetEventType :one +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types WHERE id = $1 +` + +func (q *Queries) GetEventType(ctx context.Context, id string) (EventType, error) { + row := q.db.QueryRow(ctx, getEventType, id) + var i EventType + err := row.Scan( + &i.ID, + &i.Source, + &i.Name, + &i.Rule, + &i.Schema, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + const getMapping = `-- name: GetMapping :one SELECT id, name, source_hint, fields, created_at, updated_at FROM mappings WHERE id = $1 @@ -201,8 +266,8 @@ func (q *Queries) InsertBindingDispatch(ctx context.Context, arg InsertBindingDi const insertCapture = `-- name: InsertCapture :exec -INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8) +INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ` type InsertCaptureParams struct { @@ -214,6 +279,7 @@ type InsertCaptureParams struct { Body string Authenticated bool ReceivedAt time.Time + EventType string } // EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls. @@ -227,6 +293,31 @@ func (q *Queries) InsertCapture(ctx context.Context, arg InsertCaptureParams) er arg.Body, arg.Authenticated, arg.ReceivedAt, + arg.EventType, + ) + return err +} + +const insertEventType = `-- name: InsertEventType :exec +INSERT INTO event_types (id, source, name, rule, schema) +VALUES ($1, $2, $3, $4, $5) +` + +type InsertEventTypeParams struct { + ID string + Source string + Name string + Rule string + Schema string +} + +func (q *Queries) InsertEventType(ctx context.Context, arg InsertEventTypeParams) error { + _, err := q.db.Exec(ctx, insertEventType, + arg.ID, + arg.Source, + arg.Name, + arg.Rule, + arg.Schema, ) return err } @@ -342,7 +433,7 @@ func (q *Queries) ListBindings(ctx context.Context) ([]Binding, error) { } const listCaptures = `-- name: ListCaptures :many -SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at +SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type FROM captures ORDER BY received_at DESC LIMIT $1 @@ -366,6 +457,40 @@ func (q *Queries) ListCaptures(ctx context.Context, limit int32) ([]Capture, err &i.Body, &i.Authenticated, &i.ReceivedAt, + &i.EventType, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listEventTypes = `-- name: ListEventTypes :many +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types ORDER BY source, name +` + +func (q *Queries) ListEventTypes(ctx context.Context) ([]EventType, error) { + rows, err := q.db.Query(ctx, listEventTypes) + if err != nil { + return nil, err + } + defer rows.Close() + var items []EventType + for rows.Next() { + var i EventType + if err := rows.Scan( + &i.ID, + &i.Source, + &i.Name, + &i.Rule, + &i.Schema, + &i.CreatedAt, + &i.UpdatedAt, ); err != nil { return nil, err } @@ -410,9 +535,11 @@ func (q *Queries) ListMappings(ctx context.Context) ([]Mapping, error) { } const listUndispatchedCaptures = `-- name: ListUndispatchedCaptures :many -SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at +SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type FROM captures WHERE source = ANY($1::text[]) + -- An unidentified capture is never dispatched. + AND event_type <> '' AND id NOT IN (SELECT capture FROM binding_dispatches) ORDER BY received_at DESC LIMIT $2 @@ -441,6 +568,7 @@ func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispat &i.Body, &i.Authenticated, &i.ReceivedAt, + &i.EventType, ); err != nil { return nil, err } @@ -452,6 +580,17 @@ func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispat return items, nil } +const lockEventTypeSource = `-- name: LockEventTypeSource :exec +SELECT pg_advisory_xact_lock(hashtext('event_types:' || $1::text)) +` + +// Serialises saves per source so two concurrent saves cannot each pass the +// overlap check against a set that excludes the other. +func (q *Queries) LockEventTypeSource(ctx context.Context, source string) error { + _, err := q.db.Exec(ctx, lockEventTypeSource, source) + return err +} + const setBindingArmed = `-- name: SetBindingArmed :execrows UPDATE bindings SET status = 'armed', updated_at = now() WHERE id = $1 ` @@ -549,6 +688,24 @@ func (q *Queries) UpdateBinding(ctx context.Context, arg UpdateBindingParams) (i return result.RowsAffected(), nil } +const updateEventType = `-- name: UpdateEventType :execrows +UPDATE event_types SET name = $2, rule = $3, updated_at = now() WHERE id = $1 +` + +type UpdateEventTypeParams struct { + ID string + Name string + Rule string +} + +func (q *Queries) UpdateEventType(ctx context.Context, arg UpdateEventTypeParams) (int64, error) { + result, err := q.db.Exec(ctx, updateEventType, arg.ID, arg.Name, arg.Rule) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + const updateMapping = `-- name: UpdateMapping :execrows UPDATE mappings SET name = $2, source_hint = $3, fields = $4, updated_at = now() diff --git a/internal/infrastructure/postgres/postgresdb/models.go b/internal/infrastructure/postgres/postgresdb/models.go index abb6972af..b454bb2ae 100644 --- a/internal/infrastructure/postgres/postgresdb/models.go +++ b/internal/infrastructure/postgres/postgresdb/models.go @@ -48,6 +48,7 @@ type Capture struct { Body string Authenticated bool ReceivedAt time.Time + EventType string } type ChannelStatus struct { @@ -84,6 +85,16 @@ type Event struct { Data string } +type EventType struct { + ID string + Source string + Name string + Rule string + Schema string + CreatedAt time.Time + UpdatedAt time.Time +} + type Identity struct { ID string Kind string diff --git a/internal/infrastructure/postgres/queries/edastore.sql b/internal/infrastructure/postgres/queries/edastore.sql index 54402fbf7..1aacfbad5 100644 --- a/internal/infrastructure/postgres/queries/edastore.sql +++ b/internal/infrastructure/postgres/queries/edastore.sql @@ -1,19 +1,21 @@ -- EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls. -- name: InsertCapture :exec -INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8); +INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9); -- name: ListCaptures :many -SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at +SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type FROM captures ORDER BY received_at DESC LIMIT $1; -- name: ListUndispatchedCaptures :many -SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at +SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type FROM captures WHERE source = ANY(@sources::text[]) + -- An unidentified capture is never dispatched. + AND event_type <> '' AND id NOT IN (SELECT capture FROM binding_dispatches) ORDER BY received_at DESC LIMIT @entry_limit; @@ -97,3 +99,30 @@ VALUES ($1, $2, $3, $4, $5, $6, $7); -- name: TaskToolCalls :many SELECT id, task_id, attempt, tool, result, error, called_at FROM tool_calls WHERE task_id = $1 ORDER BY called_at ASC; + +-- name: InsertEventType :exec +INSERT INTO event_types (id, source, name, rule, schema) +VALUES ($1, $2, $3, $4, $5); + +-- name: GetEventType :one +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types WHERE id = $1; + +-- name: ListEventTypes :many +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types ORDER BY source, name; + +-- name: EventTypesForSource :many +SELECT id, source, name, rule, schema, created_at, updated_at +FROM event_types WHERE source = $1 ORDER BY name; + +-- name: UpdateEventType :execrows +UPDATE event_types SET name = $2, rule = $3, updated_at = now() WHERE id = $1; + +-- name: DeleteEventType :execrows +DELETE FROM event_types WHERE id = $1; + +-- name: LockEventTypeSource :exec +-- Serialises saves per source so two concurrent saves cannot each pass the +-- overlap check against a set that excludes the other. +SELECT pg_advisory_xact_lock(hashtext('event_types:' || sqlc.arg(source)::text));