feat(eda): mappings per event type, match counts, several bindings per source

A mapping belongs to an event type and is checked against its inferred
schema on save. Each event a mapping resolves is counted once in
mapping_matches (migration 0008), read back as match_count and
last_matched_at. Bindings carry an optional CEL filter over the mapping's
parameters, checked on save. The per-source unique indexes are dropped:
every armed binding whose mapping matches the capture's event type and
whose filter admits it dispatches once, claimed in the per-binding ledger
before its task is enqueued. The State Store gains RecordMappingMatch and
the new mapping and binding fields.
This commit is contained in:
sam committed 2026-09-24 14:42:58 +10:00
1 parent d8d7c5a1f2
commit fec0ea1f4e
35 files changed
+1900 -737

No files matched your search

+3
View File
@@ -1349,6 +1349,9 @@ func (b *boot) buildDaemon() {
if bd, ok := b.stateStore.(storecontract.BindingDispatcher); ok {
b.d.BindingDispatcher = bd
}
if mm, ok := b.stateStore.(storecontract.MappingMatchRecorder); ok {
b.d.MappingMatches = mm
}
if btc, ok := b.stateStore.(storecontract.BindingTaskCreator); ok {
b.d.BindingTaskCreator = btc
}
+3
View File
@@ -258,6 +258,9 @@ func (b *boot) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
if eventTypes, ok := any(b.eda).(storecontract.EventTypeStore); ok {
deps.EventTypes = eventTypes
}
if matches, ok := any(b.eda).(storecontract.MappingMatchRecorder); ok {
deps.MappingMatches = matches
}
}
// tool_call events project into the tool_calls collection on the same
// event-capture store: this process legitimately owns both, so the
File diff suppressed because it is too large. Load diff
@@ -73,6 +73,7 @@ const (
StateStoreService_ListMappings_FullMethodName = "/state.v1.StateStoreService/ListMappings"
StateStoreService_UpdateMapping_FullMethodName = "/state.v1.StateStoreService/UpdateMapping"
StateStoreService_DeleteMapping_FullMethodName = "/state.v1.StateStoreService/DeleteMapping"
StateStoreService_RecordMappingMatch_FullMethodName = "/state.v1.StateStoreService/RecordMappingMatch"
StateStoreService_InsertEventType_FullMethodName = "/state.v1.StateStoreService/InsertEventType"
StateStoreService_UpdateEventType_FullMethodName = "/state.v1.StateStoreService/UpdateEventType"
StateStoreService_DeleteEventType_FullMethodName = "/state.v1.StateStoreService/DeleteEventType"
@@ -201,6 +202,9 @@ type StateStoreServiceClient interface {
ListMappings(ctx context.Context, in *ListMappingsRequest, opts ...grpc.CallOption) (*ListMappingsResponse, error)
UpdateMapping(ctx context.Context, in *UpdateMappingRequest, opts ...grpc.CallOption) (*UpdateMappingResponse, error)
DeleteMapping(ctx context.Context, in *DeleteMappingRequest, opts ...grpc.CallOption) (*DeleteMappingResponse, error)
// RecordMappingMatch counts one event a mapping resolved; the same
// (mapping, capture) twice counts once.
RecordMappingMatch(ctx context.Context, in *RecordMappingMatchRequest, opts ...grpc.CallOption) (*RecordMappingMatchResponse, error)
// EventType (docs/prds/event-automation.md, "Event types"). Insert and
// Update refuse an overlapping or malformed type.
InsertEventType(ctx context.Context, in *InsertEventTypeRequest, opts ...grpc.CallOption) (*InsertEventTypeResponse, error)
@@ -799,6 +803,16 @@ func (c *stateStoreServiceClient) DeleteMapping(ctx context.Context, in *DeleteM
return out, nil
}
func (c *stateStoreServiceClient) RecordMappingMatch(ctx context.Context, in *RecordMappingMatchRequest, opts ...grpc.CallOption) (*RecordMappingMatchResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(RecordMappingMatchResponse)
err := c.cc.Invoke(ctx, StateStoreService_RecordMappingMatch_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *stateStoreServiceClient) InsertEventType(ctx context.Context, in *InsertEventTypeRequest, opts ...grpc.CallOption) (*InsertEventTypeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(InsertEventTypeResponse)
@@ -1088,6 +1102,9 @@ type StateStoreServiceServer interface {
ListMappings(context.Context, *ListMappingsRequest) (*ListMappingsResponse, error)
UpdateMapping(context.Context, *UpdateMappingRequest) (*UpdateMappingResponse, error)
DeleteMapping(context.Context, *DeleteMappingRequest) (*DeleteMappingResponse, error)
// RecordMappingMatch counts one event a mapping resolved; the same
// (mapping, capture) twice counts once.
RecordMappingMatch(context.Context, *RecordMappingMatchRequest) (*RecordMappingMatchResponse, error)
// EventType (docs/prds/event-automation.md, "Event types"). Insert and
// Update refuse an overlapping or malformed type.
InsertEventType(context.Context, *InsertEventTypeRequest) (*InsertEventTypeResponse, error)
@@ -1289,6 +1306,9 @@ func (UnimplementedStateStoreServiceServer) UpdateMapping(context.Context, *Upda
func (UnimplementedStateStoreServiceServer) DeleteMapping(context.Context, *DeleteMappingRequest) (*DeleteMappingResponse, error) {
return nil, status.Error(codes.Unimplemented, "method DeleteMapping not implemented")
}
func (UnimplementedStateStoreServiceServer) RecordMappingMatch(context.Context, *RecordMappingMatchRequest) (*RecordMappingMatchResponse, error) {
return nil, status.Error(codes.Unimplemented, "method RecordMappingMatch not implemented")
}
func (UnimplementedStateStoreServiceServer) InsertEventType(context.Context, *InsertEventTypeRequest) (*InsertEventTypeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method InsertEventType not implemented")
}
@@ -2319,6 +2339,24 @@ func _StateStoreService_DeleteMapping_Handler(srv interface{}, ctx context.Conte
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_RecordMappingMatch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RecordMappingMatchRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StateStoreServiceServer).RecordMappingMatch(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: StateStoreService_RecordMappingMatch_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StateStoreServiceServer).RecordMappingMatch(ctx, req.(*RecordMappingMatchRequest))
}
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_InsertEventType_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(InsertEventTypeRequest)
if err := dec(in); err != nil {
@@ -2833,6 +2871,10 @@ var StateStoreService_ServiceDesc = grpc.ServiceDesc{
MethodName: "DeleteMapping",
Handler: _StateStoreService_DeleteMapping_Handler,
},
{
MethodName: "RecordMappingMatch",
Handler: _StateStoreService_RecordMappingMatch_Handler,
},
{
MethodName: "InsertEventType",
Handler: _StateStoreService_InsertEventType_Handler,
+55 -55
View File
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"log/slog"
"maps"
"os"
"strings"
"sync"
@@ -120,6 +121,9 @@ type Daemon struct {
// BindingStore to keep the CRUD interface under the interfacebloat
// limit. Optional: nil disables the dispatch loop.
BindingDispatcher storecontract.BindingDispatcher
// MappingMatches counts each event a mapping resolved at dispatch.
// Optional: nil leaves match counts unrecorded.
MappingMatches storecontract.MappingMatchRecorder
// BindingTaskCreator enqueues a task triggered by a binding. Split
// off TaskLifecycle so the lifecycle surface stays narrow and the
// dispatch loop does not acquire the full task-creation contract.
@@ -611,30 +615,18 @@ func (d *Daemon) cleanupExpiredStorage(ctx context.Context) {
// single cycle's worst-case latency on a backlog.
const bindingDispatchBatchLimit = 100
// dispatchBindings walks authenticated captures whose source has at
// least one armed binding, evaluates each binding's matcher against
// the capture, resolves fields through the binding's mapping, and
// enqueues a task per matched (binding, capture) pair.
// dispatchBindings walks identified captures on sources with an armed
// binding and offers each capture to every armed binding on its source.
// Each binding dispatches it at most once (dispatchOneBinding).
//
// The binding_dispatches ledger is the at-most-once guarantee across
// cycles and daemon restarts: RecordDispatch returns
// storecontract.ErrAlreadyDispatched on a duplicate (binding_id, capture_id)
// pair, and dispatchOneBinding treats that as a normal "another cycle
// raced us" outcome rather than an error.
//
// Nil Bindings, BindingDispatcher or BindingTaskCreator disables the
// loop (legacy behaviour: daemons built before t2db.4 Phase E had no
// playbook bindings, and each field's own doc comment promises this
// degrade-not-panic contract independently).
// Nil Bindings, BindingDispatcher or BindingTaskCreator disables the loop.
func (d *Daemon) dispatchBindings(ctx context.Context) {
if d.Bindings == nil || d.BindingDispatcher == nil || d.BindingTaskCreator == nil {
return
}
// Gather armed sources first so the captures query filters down to
// only relevant rows -- the table-level view over captured_events
// is otherwise unbounded, and a backlog of unrelated sources
// should never be scanned on a cycle.
// only relevant rows.
bindings, err := d.Bindings.ListBindings(ctx)
if err != nil {
d.Log.Warn("list bindings failed", "error", err)
@@ -670,19 +662,15 @@ func (d *Daemon) dispatchBindings(ctx context.Context) {
}
}
// dispatchOneBinding handles a single (binding, capture) match: resolve
// fields through the binding's mapping, enqueue a task, and record the
// dispatch in the dedup ledger. Each step degrades independently --
// a mapping lookup failure logs and returns, a required-field
// resolution failure records a binding_dispatch_failure event and
// returns without enqueueing, an enqueue failure logs and returns, and
// a RecordDispatch returning ErrAlreadyDispatched is treated as a
// benign race winner (another cycle already created the task).
//
// Owner/repo resolution currently falls back to the single configured
// repo (when exactly one is configured); binding.Owner/Repo is a
// follow-up so multi-repo deployments don't have to live with the
// ambiguity.
// dispatchOneBinding offers one capture to one binding. The binding applies
// when its mapping belongs to the capture's event type. The mapping resolves
// the payload (a required-field failure records binding_dispatch_failure and
// stops), counts the match, and the binding's filter must admit the resolved
// parameters. The dispatch is then claimed in the binding_dispatches ledger
// before the task is enqueued: a capture stays listed until every binding for
// its event type has dispatched it, so the claim is what stops a binding that
// already fired from firing again on a later cycle. A failed enqueue after the
// claim loses that dispatch, which is the at-most-once side of the trade.
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent) {
if d.Mappings == nil {
d.Log.Warn("binding dispatch: mapping store unavailable", "binding", b.ID)
@@ -697,18 +685,25 @@ func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c st
d.Log.Warn("binding dispatch: mapping missing", "binding", b.ID, "mapping", b.MappingID)
return
}
if m.EventTypeID != c.EventType {
return
}
values, failures := mapping.Resolve(m.Fields, []byte(c.Body))
if hasBlockingFailure(m.Fields, failures) {
_, _ = d.Store.InsertEvent(ctx, events.Event{
Kind: "binding_dispatch_failure",
Detail: fmt.Sprintf("binding %q (capture %q): required field failed", b.ID, c.ID),
Data: map[string]any{
"binding_id": b.ID,
"binding_version": b.Version,
"capture_id": c.ID,
"failures": failures,
},
})
d.recordDispatchFailure(ctx, b, c, "required field failed", map[string]any{"failures": failures})
return
}
if d.MappingMatches != nil {
if err := d.MappingMatches.RecordMappingMatch(ctx, m.ID, c.ID); err != nil {
d.Log.Warn("binding dispatch: record mapping match", "mapping", m.ID, "capture", c.ID, "error", err)
}
}
filter, err := binding.CompileFilter(b.Filter, m.Fields)
if err != nil {
d.recordDispatchFailure(ctx, b, c, "filter does not compile", map[string]any{"error": err.Error()})
return
}
if admitted, _ := filter.Admits(values); !admitted {
return
}
@@ -716,27 +711,32 @@ func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c st
if !ok {
return // resolveBindingRepo already logged a warning
}
if err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0); err != nil {
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
}
return
}
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
body := renderBindingBody(values, c)
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, owner, repo, title, body, b.Workflow, "", b.ID, b.Version)
if err != nil {
if _, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, owner, repo, title, body, b.Workflow, "", b.ID, b.Version); err != nil {
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
return
}
}
if err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, task.ID); err != nil {
if errors.Is(err, storecontract.ErrAlreadyDispatched) {
// Another cycle or daemon instance raced us and already
// recorded the dispatch. The duplicate task remains
// queued (delete-on-races is its own can of worms),
// but the ledger prevents the same (binding, capture)
// pair from ever being dispatched again.
return
}
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
return
// recordDispatchFailure records why a binding did not dispatch a capture.
func (d *Daemon) recordDispatchFailure(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, reason string, extra map[string]any) {
data := map[string]any{
"binding_id": b.ID,
"binding_version": b.Version,
"capture_id": c.ID,
}
maps.Copy(data, extra)
_, _ = d.Store.InsertEvent(ctx, events.Event{
Kind: "binding_dispatch_failure",
Detail: fmt.Sprintf("binding %q (capture %q): %s", b.ID, c.ID, reason),
Data: data,
})
}
// hasBlockingFailure reports whether any of the failures belongs to a
+8 -15
View File
@@ -48,7 +48,7 @@ func TestDispatchBindingsCreatesTaskFromArmedCapture(t *testing.T) {
t.Errorf("task.Status = %q, want %q", got.Status, workflow.StatusQueued)
}
assertDispatchRecorded(t, s, bindingID, got.ID)
assertDispatchRecorded(t, s, bindingID)
}
// TestDispatchBindingsUsesBindingRepoPinWithMultipleConfiguredRepos is the
@@ -78,7 +78,7 @@ func TestDispatchBindingsUsesBindingRepoPinWithMultipleConfiguredRepos(t *testin
if tasks[0].Owner != "acme" || tasks[0].Repo != "widget" {
t.Fatalf("task owner/repo = %s/%s, want acme/widget (the binding's own pin, not a guess)", tasks[0].Owner, tasks[0].Repo)
}
assertDispatchRecorded(t, s, bindingID, tasks[0].ID)
assertDispatchRecorded(t, s, bindingID)
}
// TestDispatchBindingsRefusesUnpinnedBindingWithMultipleConfiguredRepos
@@ -367,23 +367,16 @@ func newDispatchDaemonWithRepos(t *testing.T, s *dispatchStores, repos []config.
}
}
// assertDispatchRecorded reads the dedup ledger directly: the
// (binding, capture) unique index is what guarantees no double
// dispatch. Going through the SQL table rather than the public
// BindingStore surface keeps the assertion focused on the contract
// Phase E actually writes to, not on a ListDispatches helper that
// may not exist.
func assertDispatchRecorded(t *testing.T, s *dispatchStores, bindingID string, taskID int64) {
// assertDispatchRecorded reads the dedup ledger directly: the (binding,
// capture) unique index is what guarantees no double dispatch. The dispatch
// is claimed before its task exists, so the row carries no task id.
func assertDispatchRecorded(t *testing.T, s *dispatchStores, bindingID string) {
t.Helper()
var gotBindingID string
var gotTaskID int64
if err := s.EdaStore.App().DB().NewQuery(
"SELECT binding, task_id FROM binding_dispatches WHERE binding = {:binding}").
"SELECT binding FROM binding_dispatches WHERE binding = {:binding}").
Bind(map[string]any{"binding": bindingID}).
Row(&gotBindingID, &gotTaskID); err != nil {
Row(&gotBindingID); err != nil {
t.Fatalf("binding_dispatches row missing: %v", err)
}
if gotBindingID != bindingID || gotTaskID != taskID {
t.Fatalf("binding_dispatches row = (%q, %d), want (%q, %d)", gotBindingID, gotTaskID, bindingID, taskID)
}
}
@@ -0,0 +1,176 @@
package daemon
import (
"context"
"log/slog"
"path/filepath"
"testing"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/store"
)
// fakeEDA serves bindings, mappings and one capture from memory, and records
// dispatches and mapping matches the way the store's unique keys do.
type fakeEDA struct {
bindings []binding.Binding
mappings map[string]mapping.Mapping
captures []storecontract.CapturedEvent
dispatched map[[2]string]bool
matches map[[2]string]bool
}
func (f *fakeEDA) InsertMapping(context.Context, mapping.Mapping) (string, error) { return "", nil }
func (f *fakeEDA) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) {
m, ok := f.mappings[id]
if !ok {
return nil, nil
}
return &m, nil
}
func (f *fakeEDA) ListMappings(context.Context) ([]mapping.Mapping, error) { return nil, nil }
func (f *fakeEDA) UpdateMapping(context.Context, mapping.Mapping) error { return nil }
func (f *fakeEDA) DeleteMapping(context.Context, string) error { return nil }
func (f *fakeEDA) InsertBinding(context.Context, binding.Binding) (string, error) { return "", nil }
func (f *fakeEDA) GetBinding(context.Context, string) (*binding.Binding, error) { return nil, nil }
func (f *fakeEDA) ListBindings(context.Context) ([]binding.Binding, error) { return f.bindings, nil }
func (f *fakeEDA) UpdateBinding(context.Context, binding.Binding) error { return nil }
func (f *fakeEDA) DeleteBinding(context.Context, string) error { return nil }
func (f *fakeEDA) ApproveBinding(context.Context, string) error { return nil }
func (f *fakeEDA) ArmedBindingsForSource(_ context.Context, source string) ([]binding.Binding, error) {
var out []binding.Binding
for _, b := range f.bindings {
if b.Status == binding.StatusArmed && b.Matcher.Source == source {
out = append(out, b)
}
}
return out, nil
}
func (f *fakeEDA) RecordDispatch(_ context.Context, bindingID string, _ int64, captureID string, _ int64) error {
key := [2]string{bindingID, captureID}
if f.dispatched[key] {
return storecontract.ErrAlreadyDispatched
}
f.dispatched[key] = true
return nil
}
func (f *fakeEDA) ListUndispatchedCaptures(context.Context, []string, int) ([]storecontract.CapturedEvent, error) {
return f.captures, nil
}
func (f *fakeEDA) RecordMappingMatch(_ context.Context, mappingID, captureID string) error {
f.matches[[2]string{mappingID, captureID}] = true
return nil
}
func armed(id, mappingID, filter string) binding.Binding {
return binding.Binding{
ID: id, Name: id, Matcher: binding.Matcher{Source: "fw"}, MappingID: mappingID,
Filter: filter, Workflow: "implement", Status: binding.StatusArmed, Version: 1,
}
}
func severityMapping(id, eventType string) mapping.Mapping {
return mapping.Mapping{
ID: id, Name: id, EventTypeID: eventType,
Fields: []mapping.Field{{Name: "severity", Path: "severity", Type: mapping.TypeString, Required: true}},
}
}
func TestDispatchPerBindingByEventTypeAndFilter(t *testing.T) {
capture := storecontract.CapturedEvent{
ID: "c1", Source: "fw", EventType: "blocked", Authenticated: true, Body: `{"severity":"high"}`,
}
tests := []struct {
name string
bindings []binding.Binding
mappings []mapping.Mapping
wantTasks []string
wantMatches [][2]string
}{
{
name: "several bindings on one source each dispatch once",
bindings: []binding.Binding{armed("a", "m", ""), armed("b", "m", "")},
mappings: []mapping.Mapping{severityMapping("m", "blocked")},
wantTasks: []string{"a", "b"},
wantMatches: [][2]string{{"m", "c1"}},
},
{
name: "a binding for another event type does not dispatch",
bindings: []binding.Binding{armed("a", "m", "")},
mappings: []mapping.Mapping{severityMapping("m", "allowed")},
},
{
name: "a filter that excludes the event does not dispatch it",
bindings: []binding.Binding{armed("a", "m", `severity == "low"`), armed("b", "m", `severity in ["high", "critical"]`)},
mappings: []mapping.Mapping{severityMapping("m", "blocked")},
wantTasks: []string{"b"},
wantMatches: [][2]string{{"m", "c1"}},
},
{
name: "a filter that does not compile does not dispatch",
bindings: []binding.Binding{armed("a", "m", `nope ==`)},
mappings: []mapping.Mapping{severityMapping("m", "blocked")},
// The mapping still resolved the event.
wantMatches: [][2]string{{"m", "c1"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
st, err := store.Open(t.Context(), filepath.Join(t.TempDir(), "dispatch.db"))
if err != nil {
t.Fatalf("store.Open: %v", err)
}
t.Cleanup(func() { _ = st.Close() })
eda := &fakeEDA{
bindings: tt.bindings, mappings: map[string]mapping.Mapping{},
captures: []storecontract.CapturedEvent{capture},
dispatched: map[[2]string]bool{}, matches: map[[2]string]bool{},
}
for _, m := range tt.mappings {
eda.mappings[m.ID] = m
}
d := &Daemon{
Cfg: config.NewHolder(config.Config{Repos: []config.Repo{{Owner: "acme", Name: "widget"}}}),
Store: st, Mappings: eda, Bindings: eda, BindingDispatcher: eda, MappingMatches: eda,
BindingTaskCreator: st,
Log: slog.New(slog.DiscardHandler),
}
d.dispatchBindings(t.Context())
d.dispatchBindings(t.Context())
tasks, err := st.Tasks(t.Context(), 10)
if err != nil {
t.Fatalf("Tasks: %v", err)
}
got := map[string]int{}
for _, task := range tasks {
got[task.BindingID]++
}
if len(tasks) != len(tt.wantTasks) {
t.Fatalf("tasks by binding = %v, want one each for %v", got, tt.wantTasks)
}
for _, id := range tt.wantTasks {
if got[id] != 1 {
t.Fatalf("tasks by binding = %v, want one each for %v", got, tt.wantTasks)
}
}
if len(eda.matches) != len(tt.wantMatches) {
t.Fatalf("mapping matches = %v, want %v", eda.matches, tt.wantMatches)
}
for _, key := range tt.wantMatches {
if !eda.matches[key] {
t.Fatalf("mapping matches = %v, want %v", eda.matches, tt.wantMatches)
}
}
})
}
}
+6 -2
View File
@@ -34,7 +34,8 @@ type Matcher struct {
// Binding is the runtime-editable entity that ties a matcher, a payload
// mapping, and a workflow together. Bindings live in the store, not in
// config.toml, so operators can author them from the dashboard while the
// daemon runs.
// daemon runs. Any number of bindings may share a source: each one applies to
// the event type its mapping belongs to.
//
// Version is bumped on every UpdateBinding so a later edit cannot
// silently rewrite the historical provenance of a task that already
@@ -46,7 +47,10 @@ type Binding struct {
Name string `json:"name"`
Matcher Matcher `json:"matcher"`
MappingID string `json:"mapping_id"`
Workflow string `json:"workflow"`
// Filter is an optional CEL expression over the mapping's parameters
// (CompileFilter). An event it excludes is not dispatched.
Filter string `json:"filter,omitempty"`
Workflow string `json:"workflow"`
// Owner and Repo pin a binding to a specific configured repo, so a
// multi-repo deployment can dispatch correctly. Both empty means "no
// pin" -- resolveBindingRepo falls back to the single-configured-repo
+78
View File
@@ -0,0 +1,78 @@
package binding
import (
"fmt"
"strings"
"cel.dev/cel-go/cel"
"github.com/samcharles93/archie-core/internal/domain/eda/expr"
"github.com/samcharles93/archie-core/internal/domain/mapping"
)
// Filter is a compiled binding filter: a CEL expression over the mapping's
// named parameters, each declared as a variable of its field type
// (`severity in ["high", "critical"]`). A nil Filter admits every event.
type Filter struct {
prg cel.Program
}
// CompileFilter checks src against the mapping's fields and compiles it. An
// empty src is no filter. A reference to a parameter the mapping lacks, a type
// error, or a non-boolean result is refused.
func CompileFilter(src string, fields []mapping.Field) (*Filter, error) {
if strings.TrimSpace(src) == "" {
return nil, nil
}
opts := make([]cel.EnvOption, 0, len(fields))
for _, f := range fields {
opts = append(opts, cel.Variable(f.Name, celType(f.Type)))
}
env, err := cel.NewEnv(opts...)
if err != nil {
return nil, fmt.Errorf("binding: filter: %w", err)
}
ast, issues := env.Compile(src)
if issues != nil && issues.Err() != nil {
return nil, fmt.Errorf("binding: filter: %w", issues.Err())
}
if !ast.OutputType().IsExactType(cel.BoolType) {
return nil, fmt.Errorf("binding: filter must be a boolean, not %s", ast.OutputType())
}
prg, err := env.Program(ast, cel.CostLimit(expr.DefaultCostLimit))
if err != nil {
return nil, fmt.Errorf("binding: filter: %w", err)
}
return &Filter{prg: prg}, nil
}
// Admits evaluates the filter over resolved parameter values. An evaluation
// error, such as a parameter that did not resolve, excludes the event.
func (f *Filter) Admits(values map[string]any) (bool, error) {
if f == nil {
return true, nil
}
v, _, err := f.prg.Eval(values)
if err != nil {
return false, fmt.Errorf("binding: filter: %w", err)
}
admitted, _ := v.Value().(bool)
return admitted, nil
}
func celType(t mapping.FieldType) *cel.Type {
switch t {
case mapping.TypeString:
return cel.StringType
case mapping.TypeNumber:
return cel.DoubleType
case mapping.TypeBool:
return cel.BoolType
case mapping.TypeObject:
return cel.MapType(cel.StringType, cel.DynType)
case mapping.TypeArray:
return cel.ListType(cel.DynType)
default:
return cel.DynType
}
}
+59
View File
@@ -0,0 +1,59 @@
package binding
import (
"testing"
"github.com/samcharles93/archie-core/internal/domain/mapping"
)
var filterFields = []mapping.Field{
{Name: "severity", Path: "severity", Type: mapping.TypeString},
{Name: "count", Path: "count", Type: mapping.TypeNumber},
{Name: "urgent", Path: "urgent", Type: mapping.TypeBool},
{Name: "extra", Path: "extra", Type: mapping.TypeAny},
}
func TestCompileFilterRefuses(t *testing.T) {
tests := []struct{ name, src string }{
{"unknown parameter", `sender == "x"`},
{"not boolean", `severity`},
{"type mismatch", `count == "3"`},
{"syntax", `severity in [`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if _, err := CompileFilter(tt.src, filterFields); err == nil {
t.Fatalf("CompileFilter(%q) = nil error, want refusal", tt.src)
}
})
}
}
func TestFilterAdmits(t *testing.T) {
values := map[string]any{"severity": "high", "count": float64(3), "urgent": true, "extra": "x"}
tests := []struct {
name string
src string
values map[string]any
want bool
}{
{"empty filter admits everything", "", values, true},
{"list membership admits", `severity in ["high", "critical"]`, values, true},
{"list membership excludes", `severity in ["critical"]`, values, false},
{"number comparison", `count > 2.0 && urgent`, values, true},
{"dyn parameter", `extra == "x"`, values, true},
{"unresolved parameter excludes", `severity == "high"`, map[string]any{"count": float64(1)}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := CompileFilter(tt.src, filterFields)
if err != nil {
t.Fatalf("CompileFilter: %v", err)
}
got, _ := f.Admits(tt.values)
if got != tt.want {
t.Fatalf("Admits = %v, want %v", got, tt.want)
}
})
}
}
+56 -9
View File
@@ -8,10 +8,13 @@ package mapping
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
)
// FieldType is the expected JSON shape of a mapped field, checked at
@@ -46,16 +49,20 @@ type Field struct {
Required bool `json:"required"`
}
// Mapping is a named, reusable set of field bindings, authored by clicking
// through one example captured event but not pinned to that event or its
// source -- t2db.4's matcher decides which events a binding applies to.
// Mapping is a named set of field bindings belonging to one event type and
// checked against its schema (CheckSchema). Any number of bindings may reuse
// it. MatchCount and LastMatchedAt are read-only: the store counts each
// capture the mapping resolved at dispatch.
type Mapping struct {
ID string `json:"id"`
Name string `json:"name"`
SourceHint string `json:"source_hint"`
Fields []Field `json:"fields"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
ID string `json:"id"`
Name string `json:"name"`
SourceHint string `json:"source_hint"`
EventTypeID string `json:"event_type_id"`
Fields []Field `json:"fields"`
MatchCount int64 `json:"match_count"`
LastMatchedAt time.Time `json:"last_matched_at"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// validTypes is the set of FieldType values Validate accepts.
@@ -231,6 +238,46 @@ func walk(node any, path []pathSegment) (value any, found bool) {
return cur, true
}
// ErrSchema marks a mapping whose field paths or types disagree with its
// event type's inferred schema.
var ErrSchema = errors.New("mapping: field does not fit the event type's schema")
// CheckSchema refuses fields whose path the event type's schema lacks, or
// whose declared type differs from the schema's. Array indices match the
// schema's "[]" segment, and a path the example held as null accepts any type.
func CheckSchema(fields []Field, schema map[string]eventtype.ValueType) error {
for _, f := range fields {
got, ok := schema[schemaPath(f.Path)]
if !ok {
return fmt.Errorf("%w: %s: path %q is not in the event type", ErrSchema, f.Name, f.Path)
}
if f.Type != TypeAny && got != eventtype.TypeNull && string(got) != string(f.Type) {
return fmt.Errorf("%w: %s: path %q is %s, not %s", ErrSchema, f.Name, f.Path, got, f.Type)
}
}
return nil
}
// schemaPath rewrites a resolve path ("items[0].id") into the schema's form
// ("items[].id"). A malformed path has no schema form.
func schemaPath(path string) string {
var b strings.Builder
for _, seg := range parsePath(path) {
switch {
case seg.Invalid:
return ""
case seg.Index >= 0:
b.WriteString("[]")
default:
if b.Len() > 0 {
b.WriteByte('.')
}
b.WriteString(seg.Key)
}
}
return b.String()
}
func typeMatches(v any, want FieldType) bool {
if want == TypeAny {
return true
+51
View File
@@ -0,0 +1,51 @@
package mapping_test
import (
"errors"
"testing"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
"github.com/samcharles93/archie-core/internal/domain/mapping"
)
func TestCheckSchema(t *testing.T) {
schema := map[string]eventtype.ValueType{
"action": eventtype.TypeString,
"issue": eventtype.TypeObject,
"issue.number": eventtype.TypeNumber,
"labels": eventtype.TypeArray,
"labels[]": eventtype.TypeObject,
"labels[].name": eventtype.TypeString,
"closed_at": eventtype.TypeNull,
"repository.url": eventtype.TypeString,
}
tests := []struct {
name string
fields []mapping.Field
ok bool
}{
{"top level path with matching type", []mapping.Field{{Name: "a", Path: "action", Type: mapping.TypeString}}, true},
{"nested number", []mapping.Field{{Name: "n", Path: "issue.number", Type: mapping.TypeNumber}}, true},
{"indexed array element maps to the [] path", []mapping.Field{{Name: "l", Path: "labels[0].name", Type: mapping.TypeString}}, true},
{"any accepts every schema type", []mapping.Field{{Name: "i", Path: "issue", Type: mapping.TypeAny}}, true},
{"a null example value accepts any declared type", []mapping.Field{{Name: "c", Path: "closed_at", Type: mapping.TypeString}}, true},
{"path the type does not have", []mapping.Field{{Name: "x", Path: "sender.login", Type: mapping.TypeString}}, false},
{"type differs from the schema", []mapping.Field{{Name: "n", Path: "issue.number", Type: mapping.TypeString}}, false},
{"malformed index", []mapping.Field{{Name: "l", Path: "labels[x].name", Type: mapping.TypeString}}, false},
{"one bad field refuses the mapping", []mapping.Field{
{Name: "a", Path: "action", Type: mapping.TypeString},
{Name: "b", Path: "nope", Type: mapping.TypeString},
}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := mapping.CheckSchema(tt.fields, schema)
if tt.ok && err != nil {
t.Fatalf("CheckSchema() = %v, want nil", err)
}
if !tt.ok && !errors.Is(err, mapping.ErrSchema) {
t.Fatalf("CheckSchema() = %v, want ErrSchema", err)
}
})
}
}
@@ -178,6 +178,13 @@ type MappingStore interface {
DeleteMapping(ctx context.Context, id string) error
}
// MappingMatchRecorder counts the events each mapping resolved, which the
// store reports as Mapping.MatchCount and LastMatchedAt. Recording the same
// (mapping, capture) twice counts once.
type MappingMatchRecorder interface {
RecordMappingMatch(ctx context.Context, mappingID, captureID string) error
}
// EventTypeStore persists event types (docs/prds/event-automation.md, "Event
// types"). Insert and Update refuse a type whose rule overlaps another on the
// same source with eventtype.ErrOverlap, and a malformed one with
@@ -219,7 +226,8 @@ type BindingDispatcher interface {
bindingVersion int64,
captureID string,
// taskID addresses the SQLite-owned task tables, which are not
// migrating: it stays an integer on purpose.
// migrating: it stays an integer on purpose. The binding dispatch
// loop claims before it enqueues, so it records 0.
taskID int64,
) error
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
@@ -15,7 +15,6 @@ package captureintake
import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net"
@@ -92,10 +91,7 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
armed, ok := rc.armedBindings(w, r, source)
if !ok {
return
}
armed := rc.armedBindings(r, source)
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytesOrFallback(rc.MaxBodyBytes))
body, err := io.ReadAll(r.Body)
@@ -106,19 +102,22 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// HMAC verification -- only when exactly one binding is armed for this
// source. Mirrors channels/webhook/webhook.go's header precedence:
// HMAC verification against each binding armed for this source: a
// signature valid for any of their secrets authenticates the event. Mirrors channels/webhook/webhook.go's header precedence:
// GitHub-style X-Hub-Signature-256 first, X-Signature-256 fallback.
// Empty header + non-empty secret is never valid (VerifyHMAC's own
// guard), so an absent signature with an armed binding lands here as
// authenticated=false rather than authenticated=true.
sig := r.Header.Get("X-Hub-Signature-256")
if sig == "" {
sig = r.Header.Get("X-Signature-256")
}
authenticated := false
if len(armed) == 1 {
sig := r.Header.Get("X-Hub-Signature-256")
if sig == "" {
sig = r.Header.Get("X-Signature-256")
for _, b := range armed {
if webhookguard.VerifyHMAC(body, sig, b.Secret) {
authenticated = true
break
}
authenticated = webhookguard.VerifyHMAC(body, sig, armed[0].Secret)
}
headers, _ := json.Marshal(r.Header)
@@ -173,30 +172,17 @@ func (rc *Receiver) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// read. A lookup error is logged and treated as "no armed binding" -- the
// dispatch loop's auth check is the actual gate, so failing closed here would
// amplify a transient store hiccup into a capture outage for every
// sender-not-yet-bound flow. If a future phase wants fail-closed semantics,
// that is a separate decision and should not silently land here.
//
// Two armed bindings for one source would race over every inbound webhook for
// that source: the write-time overlap check should make this state
// impossible, but the dispatch loop needs a single binding per capture, so
// the overlap surfaces as 409 at capture time (belt-and-braces TOCTOU guard
// -- see the store's ApproveBinding).
func (rc *Receiver) armedBindings(w http.ResponseWriter, r *http.Request, source string) ([]binding.Binding, bool) {
// sender-not-yet-bound flow.
func (rc *Receiver) armedBindings(r *http.Request, source string) []binding.Binding {
if rc.Bindings == nil {
return nil, true
return nil
}
armed, err := rc.Bindings.ArmedBindingsForSource(r.Context(), source)
if err != nil {
rc.logger().Warn("armed bindings lookup", "source", source, "err", err)
return nil, true
return nil
}
if len(armed) > 1 {
http.Error(w,
fmt.Sprintf("multiple armed bindings for source %q -- overlap rejected", source),
http.StatusConflict)
return nil, false
}
return armed, true
return armed
}
func (rc *Receiver) logger() *slog.Logger {
@@ -75,13 +75,8 @@ func captureTestServer(t *testing.T) testReceiver {
}
// stubBindingStore injects a configurable armed-binding lookup result so
// handleCapture tests can exercise the single-binding HMAC path and the
// belt-and-braces 409-on-multiple path without seeding the real store
// (which would either run into the overlap-rejection guard or require
// bypassing the draft -> pending_approval -> armed lifecycle). The 409
// path exists exactly for the case where the write-time check was
// bypassed somehow, so simulating that case directly is the most honest
// way to cover it.
// handleCapture tests can exercise the HMAC path without seeding the real
// store through the draft -> pending_approval -> armed lifecycle.
type stubBindingStore struct {
armedBySource map[string][]binding.Binding
lookupErr error
@@ -529,37 +524,47 @@ func TestHandleCaptureWithNoSignatureMarksUnauthenticated(t *testing.T) {
}
}
// TestHandleCaptureWithMultipleArmedBindingsReturns409 covers the
// belt-and-braces TOCTOU guard: two armed bindings for the same source
// would race over every inbound webhook for that source, so the
// capture-time handler returns 409 Conflict instead of silently picking
// a winner. The store's overlap check should make this impossible in
// practice; the 409 is the safety net. The stub injects the state
// directly to exercise that safety net.
func TestHandleCaptureWithMultipleArmedBindingsReturns409(t *testing.T) {
srv := captureTestServer(t)
srv.Bindings = &stubBindingStore{
armedBySource: map[string][]binding.Binding{
"github": {
{ID: "rbind0000000001", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "abcdefghijklmnop"},
{ID: "rbind0000000002", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "qrstuvwxyz123456"},
},
},
// TestHandleCaptureWithSeveralArmedBindings: several bindings may share a
// source, and an event signed with any one of their secrets is authenticated.
func TestHandleCaptureWithSeveralArmedBindings(t *testing.T) {
tests := []struct {
name string
secret string
want bool
}{
{"first binding's secret", "abcdefghijklmnop", true},
{"second binding's secret", "qrstuvwxyz123456", true},
{"no binding's secret", "not-the-real-secret-aaaaaaaaa", false},
}
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/webhooks/capture/github", strings.NewReader(`{}`))
w := httptest.NewRecorder()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := captureTestServer(t)
srv.Bindings = &stubBindingStore{
armedBySource: map[string][]binding.Binding{
"github": {
{ID: "rbind0000000001", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "abcdefghijklmnop"},
{ID: "rbind0000000002", Matcher: binding.Matcher{Source: "github"}, Status: binding.StatusArmed, Secret: "qrstuvwxyz123456"},
},
},
}
body := `{}`
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/webhooks/capture/github", strings.NewReader(body))
req.Header.Set("X-Hub-Signature-256", hmacSHA256(tt.secret, body))
w := httptest.NewRecorder()
srv.Handler().ServeHTTP(w, req)
srv.Handler().ServeHTTP(w, req)
if w.Code != http.StatusConflict {
t.Fatalf("status = %d, want %d (multiple armed bindings must surface as 409 at capture time)", w.Code, http.StatusConflict)
}
got, err := srv.store.ListCaptures(t.Context(), 10)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 0 {
t.Fatalf("captured rows = %d, want 0 (a 409 must never persist a row)", len(got))
if w.Code != http.StatusAccepted {
t.Fatalf("status = %d, want %d", w.Code, http.StatusAccepted)
}
got, err := srv.store.ListCaptures(t.Context(), 10)
if err != nil {
t.Fatalf("ListCaptures: %v", err)
}
if len(got) != 1 || got[0].Authenticated != tt.want {
t.Fatalf("captured = %+v, want 1 row with Authenticated=%v", got, tt.want)
}
})
}
}
+5 -1
View File
@@ -25,9 +25,13 @@ var droppedColumns = map[string]bool{
// newColumns are target columns added after the legacy stores, with no
// legacy value to carry. Imported rows take the column default: an imported
// capture has no event type, so it stays unidentified and never dispatches.
// capture has no event type, so it stays unidentified and never dispatches;
// an imported mapping has no event type, so no binding using it dispatches
// until it is given one; an imported binding has no filter.
var newColumns = map[string]bool{
"captures.event_type": true,
"mappings.event_type": true,
"bindings.filter": true,
}
// targetKeys override a table's legacy key where the target has no column
+42 -33
View File
@@ -29,11 +29,12 @@ import (
// satisfies the same surface as *edastore.Store, so the cutover wave can swap
// one for the other behind storecontract without touching the consumers.
var (
_ storecontract.CaptureStore = (*EDA)(nil)
_ storecontract.MappingStore = (*EDA)(nil)
_ storecontract.BindingStore = (*EDA)(nil)
_ storecontract.BindingDispatcher = (*EDA)(nil)
_ storecontract.PlaybookDispatcher = (*EDA)(nil)
_ storecontract.CaptureStore = (*EDA)(nil)
_ storecontract.MappingStore = (*EDA)(nil)
_ storecontract.BindingStore = (*EDA)(nil)
_ storecontract.BindingDispatcher = (*EDA)(nil)
_ storecontract.MappingMatchRecorder = (*EDA)(nil)
_ storecontract.PlaybookDispatcher = (*EDA)(nil)
)
// EDA is the PostgreSQL-backed EDA persistence.
@@ -151,8 +152,8 @@ func (s *EDA) ListCaptures(ctx context.Context, limit int) ([]storecontract.Capt
}
// ListUndispatchedCaptures returns recent identified captures for the given
// sources that no binding has dispatched yet. An unidentified capture is
// never returned, so it never dispatches.
// sources that some armed binding for their event type has not dispatched yet.
// An unidentified capture is never returned, so it never dispatches.
func (s *EDA) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) {
if len(sources) == 0 || limit <= 0 {
return nil, nil
@@ -181,20 +182,27 @@ func marshalMappingFields(fields []mapping.Field) (string, error) {
return string(encoded), nil
}
func mappingValue(r postgresdb.Mapping) (mapping.Mapping, error) {
func mappingValue(r postgresdb.Mapping, matches int64, lastMatched time.Time) (mapping.Mapping, error) {
var fields []mapping.Field
if raw := r.Fields; raw != "" {
if err := json.Unmarshal([]byte(raw), &fields); err != nil {
return mapping.Mapping{}, fmt.Errorf("edastore: decode mapping fields: %w", err)
}
}
// The query reports the epoch for a mapping that never matched.
if matches == 0 {
lastMatched = time.Time{}
}
return mapping.Mapping{
ID: r.ID,
Name: r.Name,
SourceHint: r.SourceHint,
Fields: fields,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
ID: r.ID,
Name: r.Name,
SourceHint: r.SourceHint,
EventTypeID: r.EventType,
Fields: fields,
MatchCount: matches,
LastMatchedAt: lastMatched,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
}, nil
}
@@ -208,6 +216,7 @@ func (s *EDA) InsertMapping(ctx context.Context, m mapping.Mapping) (string, err
ID: id,
Name: m.Name,
SourceHint: m.SourceHint,
EventType: m.EventTypeID,
Fields: encoded,
}); err != nil {
return "", fmt.Errorf("edastore: insert mapping: %w", err)
@@ -226,7 +235,7 @@ func (s *EDA) GetMapping(ctx context.Context, id string) (*mapping.Mapping, erro
if err != nil {
return nil, err
}
m, err := mappingValue(r)
m, err := mappingValue(r.Mapping, r.MatchCount, r.LastMatchedAt)
if err != nil {
return nil, err
}
@@ -240,7 +249,7 @@ func (s *EDA) ListMappings(ctx context.Context) ([]mapping.Mapping, error) {
}
out := make([]mapping.Mapping, 0, len(rows))
for _, r := range rows {
m, err := mappingValue(r)
m, err := mappingValue(r.Mapping, r.MatchCount, r.LastMatchedAt)
if err != nil {
return nil, err
}
@@ -258,6 +267,7 @@ func (s *EDA) UpdateMapping(ctx context.Context, m mapping.Mapping) error {
ID: m.ID,
Name: m.Name,
SourceHint: m.SourceHint,
EventType: m.EventTypeID,
Fields: encoded,
})
if err != nil {
@@ -282,6 +292,16 @@ func (s *EDA) DeleteMapping(ctx context.Context, id string) error {
return nil
}
// RecordMappingMatch counts one event a mapping resolved. Recording the same
// (mapping, capture) again is a no-op, so the count rises once per event no
// matter how many bindings share the mapping.
func (s *EDA) RecordMappingMatch(ctx context.Context, mappingID, captureID string) error {
if err := s.q.InsertMappingMatch(ctx, postgresdb.InsertMappingMatchParams{Mapping: mappingID, Capture: captureID}); err != nil {
return fmt.Errorf("edastore: record mapping match: %w", err)
}
return nil
}
// --- bindings ---
func (s *EDA) encryptSecret(secret string) (string, error) {
@@ -301,6 +321,7 @@ func (s *EDA) bindingValue(r postgresdb.Binding) (binding.Binding, error) {
Name: r.Name,
Matcher: binding.Matcher{Source: r.Source},
MappingID: r.Mapping,
Filter: r.Filter,
Workflow: r.Workflow,
Owner: r.Owner,
Repo: r.Repo,
@@ -320,10 +341,8 @@ func (s *EDA) bindingValue(r postgresdb.Binding) (binding.Binding, error) {
return b, nil
}
// InsertBinding stores a new binding as pending_approval. "One binding per
// source" is a schema constraint (idx_bindings_source), so a concurrent
// second insert for the same source is refused by the database and surfaced
// here as ErrBindingOverlap.
// InsertBinding stores a new binding as pending_approval. Any number of
// bindings may share a source.
func (s *EDA) InsertBinding(ctx context.Context, b binding.Binding) (string, error) {
secret := b.Secret
if secret != "" {
@@ -339,15 +358,13 @@ func (s *EDA) InsertBinding(ctx context.Context, b binding.Binding) (string, err
Name: b.Name,
Source: b.Matcher.Source,
Mapping: b.MappingID,
Filter: b.Filter,
Workflow: b.Workflow,
Owner: b.Owner,
Repo: b.Repo,
Status: string(binding.StatusPendingApproval),
Secret: secret,
})
if isUniqueViolation(err) {
return "", storecontract.ErrBindingOverlap
}
if err != nil {
return "", fmt.Errorf("edastore: insert binding: %w", err)
}
@@ -423,15 +440,13 @@ func (s *EDA) UpdateBinding(ctx context.Context, b binding.Binding) error {
Name: b.Name,
Source: b.Matcher.Source,
Mapping: b.MappingID,
Filter: b.Filter,
Workflow: b.Workflow,
Owner: b.Owner,
Repo: b.Repo,
Status: string(binding.StatusPendingApproval),
Secret: secret,
})
if isUniqueViolation(err) {
return storecontract.ErrBindingOverlap
}
if err != nil {
return err
}
@@ -454,10 +469,7 @@ func (s *EDA) DeleteBinding(ctx context.Context, id string) error {
return nil
}
// ApproveBinding is the only transition that arms a binding. "One armed
// binding per source" is a partial unique constraint (idx_bindings_armed_source),
// so a concurrent approval of a sibling binding on the same source is refused
// by the database and surfaced here as ErrBindingOverlap.
// ApproveBinding is the only transition that arms a binding.
func (s *EDA) ApproveBinding(ctx context.Context, id string) error {
r, err := s.q.GetBinding(ctx, id)
if errors.Is(err, pgx.ErrNoRows) {
@@ -470,9 +482,6 @@ func (s *EDA) ApproveBinding(ctx context.Context, id string) error {
return storecontract.ErrBindingTransition
}
n, err := s.q.SetBindingArmed(ctx, id)
if isUniqueViolation(err) {
return storecontract.ErrBindingOverlap
}
if err != nil {
return err
}
@@ -0,0 +1,166 @@
package postgres
import (
"testing"
"time"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
)
// eventTypeFixture saves a type on source matching events whose "kind" equals kind.
func eventTypeFixture(t *testing.T, s *EDA, source, kind string) string {
t.Helper()
id, err := s.InsertEventType(t.Context(), eventtype.EventType{
Source: source, Name: kind,
Rule: eventtype.Rule{Payload: []eventtype.PayloadCondition{{Path: "kind", Op: eventtype.OpEquals, Value: kind}}},
})
if err != nil {
t.Fatalf("InsertEventType() error = %v", err)
}
return id
}
func armedBinding(t *testing.T, s *EDA, source, mappingID string) string {
t.Helper()
id, err := s.InsertBinding(t.Context(), binding.Binding{
Name: "n", Matcher: binding.Matcher{Source: source}, MappingID: mappingID, Workflow: "implement",
})
if err != nil {
t.Fatalf("InsertBinding() error = %v", err)
}
if err := s.ApproveBinding(t.Context(), id); err != nil {
t.Fatalf("ApproveBinding() error = %v", err)
}
return id
}
func captureOf(t *testing.T, s *EDA, source, body string) string {
t.Helper()
id, err := s.InsertCapture(t.Context(), storecontract.CapturedEvent{Source: source, Body: body}, 0, 0)
if err != nil {
t.Fatalf("InsertCapture() error = %v", err)
}
return id
}
func undispatchedIDs(t *testing.T, s *EDA, source string) map[string]bool {
t.Helper()
list, err := s.ListUndispatchedCaptures(t.Context(), []string{source}, 10)
if err != nil {
t.Fatalf("ListUndispatchedCaptures() error = %v", err)
}
out := map[string]bool{}
for _, c := range list {
out[c.ID] = true
}
return out
}
func TestSeveralArmedBindingsShareASource(t *testing.T) {
s := edaFor(t)
a := armedBinding(t, s, "github", "")
b := armedBinding(t, s, "github", "")
armed, err := s.ArmedBindingsForSource(t.Context(), "github")
if err != nil {
t.Fatalf("ArmedBindingsForSource() error = %v", err)
}
got := map[string]bool{}
for _, x := range armed {
got[x.ID] = true
}
if !got[a] || !got[b] || len(armed) != 2 {
t.Fatalf("armed = %v, want both %s and %s", got, a, b)
}
}
func TestMappingAndBindingRoundTripEventTypeAndFilter(t *testing.T) {
s := edaFor(t)
typeID := eventTypeFixture(t, s, "github", "push")
mid, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m", EventTypeID: typeID})
if err != nil {
t.Fatalf("InsertMapping() error = %v", err)
}
m, err := s.GetMapping(t.Context(), mid)
if err != nil || m.EventTypeID != typeID {
t.Fatalf("GetMapping() event type = %q (err %v), want %q", m.EventTypeID, err, typeID)
}
bid, err := s.InsertBinding(t.Context(), binding.Binding{
Name: "n", Matcher: binding.Matcher{Source: "github"}, MappingID: mid, Workflow: "implement",
Filter: `kind == "push"`,
})
if err != nil {
t.Fatalf("InsertBinding() error = %v", err)
}
b, err := s.GetBinding(t.Context(), bid)
if err != nil || b.Filter != `kind == "push"` {
t.Fatalf("GetBinding() filter = %q (err %v)", b.Filter, err)
}
b.Filter = ""
if err := s.UpdateBinding(t.Context(), *b); err != nil {
t.Fatalf("UpdateBinding() error = %v", err)
}
if b, _ = s.GetBinding(t.Context(), bid); b.Filter != "" {
t.Fatalf("filter after clearing = %q, want empty", b.Filter)
}
}
func TestMappingMatchCountRisesOncePerEvent(t *testing.T) {
s := edaFor(t)
mid, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m"})
if err != nil {
t.Fatalf("InsertMapping() error = %v", err)
}
before := time.Now().Add(-time.Minute)
for _, capture := range []string{"c1", "c2", "c1"} {
if err := s.RecordMappingMatch(t.Context(), mid, capture); err != nil {
t.Fatalf("RecordMappingMatch(%s) error = %v", capture, err)
}
}
list, err := s.ListMappings(t.Context())
if err != nil || len(list) != 1 {
t.Fatalf("ListMappings() = %d (err %v)", len(list), err)
}
if list[0].MatchCount != 2 {
t.Errorf("MatchCount = %d, want 2 (one per distinct event)", list[0].MatchCount)
}
if !list[0].LastMatchedAt.After(before) {
t.Errorf("LastMatchedAt = %v, want a recent time", list[0].LastMatchedAt)
}
got, _ := s.GetMapping(t.Context(), mid)
if got.MatchCount != 2 {
t.Errorf("GetMapping MatchCount = %d, want 2", got.MatchCount)
}
}
// A capture stays undispatched until every armed binding for its event type
// has dispatched it; a binding for another event type does not hold it.
func TestUndispatchedIsPerBinding(t *testing.T) {
s := edaFor(t)
push := eventTypeFixture(t, s, "github", "push")
issue := eventTypeFixture(t, s, "github", "issue")
pushMap, _ := s.InsertMapping(t.Context(), mapping.Mapping{Name: "p", EventTypeID: push})
issueMap, _ := s.InsertMapping(t.Context(), mapping.Mapping{Name: "i", EventTypeID: issue})
a := armedBinding(t, s, "github", pushMap)
b := armedBinding(t, s, "github", pushMap)
c := armedBinding(t, s, "github", issueMap)
capture := captureOf(t, s, "github", `{"kind":"push"}`)
if !undispatchedIDs(t, s, "github")[capture] {
t.Fatal("new push capture not listed")
}
if err := s.RecordDispatch(t.Context(), a, 1, capture, 1); err != nil {
t.Fatalf("RecordDispatch(a) error = %v", err)
}
if !undispatchedIDs(t, s, "github")[capture] {
t.Fatal("capture dropped after one of two push bindings dispatched it")
}
if err := s.RecordDispatch(t.Context(), b, 1, capture, 2); err != nil {
t.Fatalf("RecordDispatch(b) error = %v", err)
}
if undispatchedIDs(t, s, "github")[capture] {
t.Fatalf("capture still listed after both push bindings dispatched it (issue binding %s must not hold it)", c)
}
}
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
)
@@ -142,6 +143,11 @@ func TestPastedTypeIdentifiesLaterCaptures(t *testing.T) {
t.Errorf("unmatched capture event type = %q, want unidentified", byBody[`{"ref":"main"}`])
}
mid, err := s.InsertMapping(t.Context(), mapping.Mapping{Name: "m", EventTypeID: id})
if err != nil {
t.Fatalf("InsertMapping() error = %v", err)
}
armedBinding(t, s, "gh", mid)
undispatched, err := s.ListUndispatchedCaptures(t.Context(), []string{"gh"}, 10)
if err != nil {
t.Fatalf("ListUndispatchedCaptures() error = %v", err)
@@ -9,7 +9,6 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/events"
@@ -92,42 +91,6 @@ func TestPlaybookDispatchIsIdempotent(t *testing.T) {
}
}
// Hazard 3: "one binding per source" is a schema constraint, so concurrent
// inserts for one source produce exactly one success and one ErrBindingOverlap
// rather than two bindings that a single SQLite writer would have serialised.
func TestConcurrentInsertBindingSameSource(t *testing.T) {
s := edaFor(t)
type result struct{ err error }
start := make(chan struct{})
results := make(chan result, 2)
for range 2 {
go func() {
<-start
_, err := s.InsertBinding(t.Context(), binding.Binding{
Name: "n", Matcher: binding.Matcher{Source: "sentry"}, Workflow: "implement",
})
results <- result{err: err}
}()
}
close(start)
var succeeded, overlapped int
for range 2 {
outcome := <-results
switch {
case outcome.err == nil:
succeeded++
case errors.Is(outcome.err, storecontract.ErrBindingOverlap):
overlapped++
default:
t.Fatalf("concurrent InsertBinding error = %v", outcome.err)
}
}
if succeeded != 1 || overlapped != 1 {
t.Fatalf("concurrent inserts = %d succeeded, %d overlapped; want 1 each", succeeded, overlapped)
}
}
// Hazard 1: UpdateBinding with an empty secret must leave the stored envelope
// untouched, so an edit form that does not echo the secret cannot blank an
// armed binding's HMAC key.
@@ -219,13 +182,9 @@ func TestBindingLifecycle(t *testing.T) {
}
}
// A capture round-trips its payload verbatim, and a dispatched capture drops
// out of the undispatched listing.
func TestCaptureRoundTripAndUndispatched(t *testing.T) {
// A capture round-trips its payload verbatim.
func TestCaptureRoundTrip(t *testing.T) {
s := edaFor(t)
if _, err := s.InsertEventType(t.Context(), eventtype.EventType{Source: "sentry", Name: "any"}); err != nil {
t.Fatalf("InsertEventType() error = %v", err)
}
body := `{"action":"created"}`
if _, err := s.InsertCapture(t.Context(), storecontract.CapturedEvent{
Source: "sentry", Body: body, Headers: `{"X-Hook":"1"}`, ContentType: "application/json",
@@ -239,19 +198,6 @@ func TestCaptureRoundTripAndUndispatched(t *testing.T) {
if list[0].Body != body {
t.Errorf("Body round-trip = %q, want %q", list[0].Body, body)
}
undispatched, err := s.ListUndispatchedCaptures(t.Context(), []string{"sentry"}, 10)
if err != nil || len(undispatched) != 1 {
t.Fatalf("ListUndispatchedCaptures() = %d (err %v), want 1", len(undispatched), err)
}
// Dispatch it; the capture must no longer be undispatched.
if err := s.RecordDispatch(t.Context(), "b1", 1, list[0].ID, 42); err != nil {
t.Fatalf("RecordDispatch() error = %v", err)
}
undispatched, err = s.ListUndispatchedCaptures(t.Context(), []string{"sentry"}, 10)
if err != nil || len(undispatched) != 0 {
t.Fatalf("ListUndispatchedCaptures() after dispatch = %d (err %v), want 0", len(undispatched), err)
}
}
func TestMappingCRUD(t *testing.T) {
@@ -0,0 +1,34 @@
-- Mappings per event type, match counts and several bindings per source
-- (docs/prds/event-automation.md, "Mappings" and "Bindings").
--
-- mappings.event_type is the event type a mapping belongs to; like
-- captures.event_type it is plain text, not a foreign key. A binding applies
-- to its mapping's event type, so any number of bindings may share a source
-- and the two per-source unique indexes are dropped.
--
-- mapping_matches holds one row per (mapping, capture) the mapping resolved;
-- a mapping's matched-event count and last match are read from it.
-- bindings.filter is an optional CEL expression over the mapping's parameters.
-- +goose Up
ALTER TABLE mappings ADD COLUMN event_type text NOT NULL DEFAULT '';
ALTER TABLE bindings ADD COLUMN filter text NOT NULL DEFAULT '';
DROP INDEX idx_bindings_source;
DROP INDEX idx_bindings_armed_source;
CREATE TABLE mapping_matches (
mapping text NOT NULL,
capture text NOT NULL,
matched_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (mapping, capture)
);
-- +goose Down
DROP TABLE mapping_matches;
CREATE UNIQUE INDEX idx_bindings_armed_source ON bindings (source) WHERE status = 'armed';
CREATE UNIQUE INDEX idx_bindings_source ON bindings (source);
ALTER TABLE bindings DROP COLUMN filter;
ALTER TABLE mappings DROP COLUMN event_type;
@@ -11,7 +11,7 @@ import (
)
const armedBindingsForSource = `-- name: ArmedBindingsForSource :many
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings WHERE source = $1 AND status = 'armed' ORDER BY created_at DESC
`
@@ -37,6 +37,7 @@ func (q *Queries) ArmedBindingsForSource(ctx context.Context, source string) ([]
&i.Secret,
&i.CreatedAt,
&i.UpdatedAt,
&i.Filter,
); err != nil {
return nil, err
}
@@ -147,7 +148,7 @@ func (q *Queries) EventTypesForSource(ctx context.Context, source string) ([]Eve
}
const getBinding = `-- name: GetBinding :one
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings WHERE id = $1
`
@@ -167,6 +168,7 @@ func (q *Queries) GetBinding(ctx context.Context, id string) (Binding, error) {
&i.Secret,
&i.CreatedAt,
&i.UpdatedAt,
&i.Filter,
)
return i, err
}
@@ -192,27 +194,38 @@ func (q *Queries) GetEventType(ctx context.Context, id string) (EventType, error
}
const getMapping = `-- name: GetMapping :one
SELECT id, name, source_hint, fields, created_at, updated_at
SELECT mappings.id, mappings.name, mappings.source_hint, mappings.fields, mappings.created_at, mappings.updated_at, mappings.event_type,
(SELECT count(*) FROM mapping_matches mm WHERE mm.mapping = mappings.id)::bigint AS match_count,
COALESCE((SELECT max(mm.matched_at) FROM mapping_matches mm WHERE mm.mapping = mappings.id), 'epoch')::timestamptz AS last_matched_at
FROM mappings WHERE id = $1
`
func (q *Queries) GetMapping(ctx context.Context, id string) (Mapping, error) {
type GetMappingRow struct {
Mapping Mapping
MatchCount int64
LastMatchedAt time.Time
}
func (q *Queries) GetMapping(ctx context.Context, id string) (GetMappingRow, error) {
row := q.db.QueryRow(ctx, getMapping, id)
var i Mapping
var i GetMappingRow
err := row.Scan(
&i.ID,
&i.Name,
&i.SourceHint,
&i.Fields,
&i.CreatedAt,
&i.UpdatedAt,
&i.Mapping.ID,
&i.Mapping.Name,
&i.Mapping.SourceHint,
&i.Mapping.Fields,
&i.Mapping.CreatedAt,
&i.Mapping.UpdatedAt,
&i.Mapping.EventType,
&i.MatchCount,
&i.LastMatchedAt,
)
return i, err
}
const insertBinding = `-- name: InsertBinding :exec
INSERT INTO bindings (id, name, source, mapping, workflow, owner, repo, version, status, secret)
VALUES ($1, $2, $3, $4, $5, $6, $7, 1, $8, $9)
INSERT INTO bindings (id, name, source, mapping, filter, workflow, owner, repo, version, status, secret)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 1, $9, $10)
`
type InsertBindingParams struct {
@@ -220,6 +233,7 @@ type InsertBindingParams struct {
Name string
Source string
Mapping string
Filter string
Workflow string
Owner string
Repo string
@@ -233,6 +247,7 @@ func (q *Queries) InsertBinding(ctx context.Context, arg InsertBindingParams) er
arg.Name,
arg.Source,
arg.Mapping,
arg.Filter,
arg.Workflow,
arg.Owner,
arg.Repo,
@@ -323,14 +338,15 @@ func (q *Queries) InsertEventType(ctx context.Context, arg InsertEventTypeParams
}
const insertMapping = `-- name: InsertMapping :exec
INSERT INTO mappings (id, name, source_hint, fields)
VALUES ($1, $2, $3, $4)
INSERT INTO mappings (id, name, source_hint, event_type, fields)
VALUES ($1, $2, $3, $4, $5)
`
type InsertMappingParams struct {
ID string
Name string
SourceHint string
EventType string
Fields string
}
@@ -339,11 +355,27 @@ func (q *Queries) InsertMapping(ctx context.Context, arg InsertMappingParams) er
arg.ID,
arg.Name,
arg.SourceHint,
arg.EventType,
arg.Fields,
)
return err
}
const insertMappingMatch = `-- name: InsertMappingMatch :exec
INSERT INTO mapping_matches (mapping, capture) VALUES ($1, $2)
ON CONFLICT DO NOTHING
`
type InsertMappingMatchParams struct {
Mapping string
Capture string
}
func (q *Queries) InsertMappingMatch(ctx context.Context, arg InsertMappingMatchParams) error {
_, err := q.db.Exec(ctx, insertMappingMatch, arg.Mapping, arg.Capture)
return err
}
const insertPlaybookDispatch = `-- name: InsertPlaybookDispatch :exec
INSERT INTO playbook_dispatches (playbook_id, playbook_version, event_id, action_id)
VALUES ($1, $2, $3, $4)
@@ -395,7 +427,7 @@ func (q *Queries) InsertToolCall(ctx context.Context, arg InsertToolCallParams)
}
const listBindings = `-- name: ListBindings :many
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings ORDER BY created_at DESC
`
@@ -421,6 +453,7 @@ func (q *Queries) ListBindings(ctx context.Context) ([]Binding, error) {
&i.Secret,
&i.CreatedAt,
&i.UpdatedAt,
&i.Filter,
); err != nil {
return nil, err
}
@@ -503,26 +536,37 @@ func (q *Queries) ListEventTypes(ctx context.Context) ([]EventType, error) {
}
const listMappings = `-- name: ListMappings :many
SELECT id, name, source_hint, fields, created_at, updated_at
SELECT mappings.id, mappings.name, mappings.source_hint, mappings.fields, mappings.created_at, mappings.updated_at, mappings.event_type,
(SELECT count(*) FROM mapping_matches mm WHERE mm.mapping = mappings.id)::bigint AS match_count,
COALESCE((SELECT max(mm.matched_at) FROM mapping_matches mm WHERE mm.mapping = mappings.id), 'epoch')::timestamptz AS last_matched_at
FROM mappings ORDER BY created_at DESC
`
func (q *Queries) ListMappings(ctx context.Context) ([]Mapping, error) {
type ListMappingsRow struct {
Mapping Mapping
MatchCount int64
LastMatchedAt time.Time
}
func (q *Queries) ListMappings(ctx context.Context) ([]ListMappingsRow, error) {
rows, err := q.db.Query(ctx, listMappings)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Mapping
var items []ListMappingsRow
for rows.Next() {
var i Mapping
var i ListMappingsRow
if err := rows.Scan(
&i.ID,
&i.Name,
&i.SourceHint,
&i.Fields,
&i.CreatedAt,
&i.UpdatedAt,
&i.Mapping.ID,
&i.Mapping.Name,
&i.Mapping.SourceHint,
&i.Mapping.Fields,
&i.Mapping.CreatedAt,
&i.Mapping.UpdatedAt,
&i.Mapping.EventType,
&i.MatchCount,
&i.LastMatchedAt,
); err != nil {
return nil, err
}
@@ -535,13 +579,16 @@ func (q *Queries) ListMappings(ctx context.Context) ([]Mapping, error) {
}
const listUndispatchedCaptures = `-- name: ListUndispatchedCaptures :many
SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type
FROM captures
WHERE source = ANY($1::text[])
-- An unidentified capture is never dispatched.
AND event_type <> ''
AND id NOT IN (SELECT capture FROM binding_dispatches)
ORDER BY received_at DESC
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.event_type
FROM captures c
WHERE c.source = ANY($1::text[])
AND c.event_type <> ''
AND EXISTS (
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
WHERE b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.capture = c.id)
)
ORDER BY c.received_at DESC
LIMIT $2
`
@@ -550,6 +597,9 @@ type ListUndispatchedCapturesParams struct {
EntryLimit int32
}
// A capture is undispatched while some armed binding on its source, whose
// mapping belongs to the capture's event type, has not dispatched it. An
// unidentified capture has no event type, so it is never listed.
func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispatchedCapturesParams) ([]Capture, error) {
rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.Sources, arg.EntryLimit)
if err != nil {
@@ -648,9 +698,9 @@ func (q *Queries) TaskToolCalls(ctx context.Context, taskID int64) ([]TaskToolCa
const updateBinding = `-- name: UpdateBinding :execrows
UPDATE bindings
SET name = $2, source = $3, mapping = $4, workflow = $5, owner = $6, repo = $7,
version = version + 1, status = $8,
secret = CASE WHEN $9::text = '' THEN secret ELSE $9 END,
SET name = $2, source = $3, mapping = $4, filter = $5, workflow = $6, owner = $7, repo = $8,
version = version + 1, status = $9,
secret = CASE WHEN $10::text = '' THEN secret ELSE $10 END,
updated_at = now()
WHERE id = $1
`
@@ -660,6 +710,7 @@ type UpdateBindingParams struct {
Name string
Source string
Mapping string
Filter string
Workflow string
Owner string
Repo string
@@ -676,6 +727,7 @@ func (q *Queries) UpdateBinding(ctx context.Context, arg UpdateBindingParams) (i
arg.Name,
arg.Source,
arg.Mapping,
arg.Filter,
arg.Workflow,
arg.Owner,
arg.Repo,
@@ -708,7 +760,7 @@ func (q *Queries) UpdateEventType(ctx context.Context, arg UpdateEventTypeParams
const updateMapping = `-- name: UpdateMapping :execrows
UPDATE mappings
SET name = $2, source_hint = $3, fields = $4, updated_at = now()
SET name = $2, source_hint = $3, event_type = $4, fields = $5, updated_at = now()
WHERE id = $1
`
@@ -716,6 +768,7 @@ type UpdateMappingParams struct {
ID string
Name string
SourceHint string
EventType string
Fields string
}
@@ -724,6 +777,7 @@ func (q *Queries) UpdateMapping(ctx context.Context, arg UpdateMappingParams) (i
arg.ID,
arg.Name,
arg.SourceHint,
arg.EventType,
arg.Fields,
)
if err != nil {
@@ -29,6 +29,7 @@ type Binding struct {
Secret string
CreatedAt time.Time
UpdatedAt time.Time
Filter string
}
type BindingDispatch struct {
@@ -144,6 +145,13 @@ type Mapping struct {
Fields string
CreatedAt time.Time
UpdatedAt time.Time
EventType string
}
type MappingMatch struct {
Mapping string
Capture string
MatchedAt time.Time
}
type Message struct {
@@ -11,13 +11,19 @@ ORDER BY received_at DESC
LIMIT $1;
-- name: ListUndispatchedCaptures :many
SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, event_type
FROM captures
WHERE source = ANY(@sources::text[])
-- An unidentified capture is never dispatched.
AND event_type <> ''
AND id NOT IN (SELECT capture FROM binding_dispatches)
ORDER BY received_at DESC
-- A capture is undispatched while some armed binding on its source, whose
-- mapping belongs to the capture's event type, has not dispatched it. An
-- unidentified capture has no event type, so it is never listed.
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.event_type
FROM captures c
WHERE c.source = ANY(@sources::text[])
AND c.event_type <> ''
AND EXISTS (
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
WHERE b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.capture = c.id)
)
ORDER BY c.received_at DESC
LIMIT @entry_limit;
-- name: DeleteCapturesOlderThan :exec
@@ -29,39 +35,47 @@ DELETE FROM captures WHERE id NOT IN (
);
-- name: InsertMapping :exec
INSERT INTO mappings (id, name, source_hint, fields)
VALUES ($1, $2, $3, $4);
INSERT INTO mappings (id, name, source_hint, event_type, fields)
VALUES ($1, $2, $3, $4, $5);
-- name: GetMapping :one
SELECT id, name, source_hint, fields, created_at, updated_at
SELECT sqlc.embed(mappings),
(SELECT count(*) FROM mapping_matches mm WHERE mm.mapping = mappings.id)::bigint AS match_count,
COALESCE((SELECT max(mm.matched_at) FROM mapping_matches mm WHERE mm.mapping = mappings.id), 'epoch')::timestamptz AS last_matched_at
FROM mappings WHERE id = $1;
-- name: ListMappings :many
SELECT id, name, source_hint, fields, created_at, updated_at
SELECT sqlc.embed(mappings),
(SELECT count(*) FROM mapping_matches mm WHERE mm.mapping = mappings.id)::bigint AS match_count,
COALESCE((SELECT max(mm.matched_at) FROM mapping_matches mm WHERE mm.mapping = mappings.id), 'epoch')::timestamptz AS last_matched_at
FROM mappings ORDER BY created_at DESC;
-- name: UpdateMapping :execrows
UPDATE mappings
SET name = $2, source_hint = $3, fields = $4, updated_at = now()
SET name = $2, source_hint = $3, event_type = $4, fields = $5, updated_at = now()
WHERE id = $1;
-- name: InsertMappingMatch :exec
INSERT INTO mapping_matches (mapping, capture) VALUES ($1, $2)
ON CONFLICT DO NOTHING;
-- name: DeleteMapping :execrows
DELETE FROM mappings WHERE id = $1;
-- name: InsertBinding :exec
INSERT INTO bindings (id, name, source, mapping, workflow, owner, repo, version, status, secret)
VALUES ($1, $2, $3, $4, $5, $6, $7, 1, $8, $9);
INSERT INTO bindings (id, name, source, mapping, filter, workflow, owner, repo, version, status, secret)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 1, $9, $10);
-- name: GetBinding :one
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings WHERE id = $1;
-- name: ListBindings :many
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings ORDER BY created_at DESC;
-- name: ArmedBindingsForSource :many
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at
SELECT id, name, source, mapping, workflow, owner, repo, version, status, secret, created_at, updated_at, filter
FROM bindings WHERE source = $1 AND status = 'armed' ORDER BY created_at DESC;
-- name: UpdateBinding :execrows
@@ -69,8 +83,8 @@ FROM bindings WHERE source = $1 AND status = 'armed' ORDER BY created_at DESC;
-- untouched so an edit form that does not echo the secret cannot blank an
-- armed binding's HMAC key.
UPDATE bindings
SET name = $2, source = $3, mapping = $4, workflow = $5, owner = $6, repo = $7,
version = version + 1, status = $8,
SET name = $2, source = $3, mapping = $4, filter = $5, workflow = $6, owner = $7, repo = $8,
version = version + 1, status = $9,
secret = CASE WHEN sqlc.arg(secret)::text = '' THEN secret ELSE sqlc.arg(secret) END,
updated_at = now()
WHERE id = $1;
+19 -13
View File
@@ -41,19 +41,20 @@ func (c *Client) ControlPlane() controlpb.ControlPlaneServiceClient { return c.c
func (c *Client) Close() error { return nil }
var (
_ task.Store = (*Client)(nil)
_ storecontract.TaskStore = (*Client)(nil)
_ storecontract.TaskLogStore = (*Client)(nil)
_ storecontract.CaptureStore = (*Client)(nil)
_ storecontract.MappingStore = (*Client)(nil)
_ storecontract.BindingStore = (*Client)(nil)
_ storecontract.BindingDispatcher = (*Client)(nil)
_ storecontract.BindingTaskCreator = (*Client)(nil)
_ storecontract.PlaybookDispatcher = (*Client)(nil)
_ storecontract.ConfigSnapshotStore = (*Client)(nil)
_ storecontract.ChannelStatusStore = (*Client)(nil)
_ storecontract.ApplyStatusStore = (*Client)(nil)
_ identity.Repository = (*Client)(nil)
_ task.Store = (*Client)(nil)
_ storecontract.TaskStore = (*Client)(nil)
_ storecontract.TaskLogStore = (*Client)(nil)
_ storecontract.CaptureStore = (*Client)(nil)
_ storecontract.MappingStore = (*Client)(nil)
_ storecontract.BindingStore = (*Client)(nil)
_ storecontract.BindingDispatcher = (*Client)(nil)
_ storecontract.MappingMatchRecorder = (*Client)(nil)
_ storecontract.BindingTaskCreator = (*Client)(nil)
_ storecontract.PlaybookDispatcher = (*Client)(nil)
_ storecontract.ConfigSnapshotStore = (*Client)(nil)
_ storecontract.ChannelStatusStore = (*Client)(nil)
_ storecontract.ApplyStatusStore = (*Client)(nil)
_ identity.Repository = (*Client)(nil)
)
// Lifecycle
@@ -415,6 +416,11 @@ func (c *Client) DeleteMapping(ctx context.Context, id string) error {
return unmapError(err)
}
func (c *Client) RecordMappingMatch(ctx context.Context, mappingID, captureID string) error {
_, err := c.client.RecordMappingMatch(ctx, &pb.RecordMappingMatchRequest{MappingId: mappingID, CaptureId: captureID})
return unmapError(err)
}
// Binding
func (c *Client) InsertBinding(ctx context.Context, b binding.Binding) (string, error) {
@@ -0,0 +1,58 @@
package staterpc
import (
"context"
"net"
"reflect"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/test/bufconn"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/mapping"
)
type fakeMappingMatches struct{ mappingID, captureID string }
func (f *fakeMappingMatches) RecordMappingMatch(_ context.Context, mappingID, captureID string) error {
f.mappingID, f.captureID = mappingID, captureID
return nil
}
func TestRecordMappingMatchCrossesTheWire(t *testing.T) {
fake := &fakeMappingMatches{}
listener := bufconn.Listen(1 << 20)
server := grpc.NewServer()
RegisterServer(server, Deps{MappingMatches: fake})
go func() { _ = server.Serve(listener) }()
t.Cleanup(func() { server.Stop(); _ = listener.Close() })
conn, err := grpc.NewClient("passthrough:///state",
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return listener.DialContext(ctx) }))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = conn.Close() })
if err := NewClient(conn).RecordMappingMatch(t.Context(), "m1", "c1"); err != nil {
t.Fatalf("RecordMappingMatch() error = %v", err)
}
if fake.mappingID != "m1" || fake.captureID != "c1" {
t.Fatalf("server received (%q, %q), want (m1, c1)", fake.mappingID, fake.captureID)
}
}
func TestMappingAndBindingCarryEventTypeCountsAndFilter(t *testing.T) {
stamp := time.Date(2026, 9, 24, 9, 0, 0, 0, time.UTC)
m := mapping.Mapping{ID: "m", Name: "n", Fields: []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}}, EventTypeID: "et-1", MatchCount: 7, LastMatchedAt: stamp, CreatedAt: stamp, UpdatedAt: stamp}
if got := mappingValue(mappingProto(m)); !reflect.DeepEqual(got, m) {
t.Fatalf("mapping after round trip = %+v, want %+v", got, m)
}
b := binding.Binding{ID: "b", Filter: `severity == "high"`, CreatedAt: stamp, UpdatedAt: stamp}
if got := bindingValue(bindingProto(b)); got.Filter != b.Filter {
t.Fatalf("binding filter after round trip = %q, want %q", got.Filter, b.Filter)
}
}
+17 -4
View File
@@ -73,10 +73,13 @@ type Deps struct {
// ChannelStatus is the channel runtime state the process hosting the channels
// reports. Optional: nil disables the pair with codes.Unavailable, which is
// the honest answer for a store service no messaging process is writing to.
ChannelStatus storecontract.ChannelStatusStore
ApplyStatus storecontract.ApplyStatusStore
Mappings storecontract.MappingStore
EventTypes storecontract.EventTypeStore
ChannelStatus storecontract.ChannelStatusStore
ApplyStatus storecontract.ApplyStatusStore
Mappings storecontract.MappingStore
EventTypes storecontract.EventTypeStore
// MappingMatches counts the events each mapping resolved. Optional: nil
// answers RecordMappingMatch with codes.Unavailable.
MappingMatches storecontract.MappingMatchRecorder
Bindings storecontract.BindingStore
BindingDispatcher storecontract.BindingDispatcher
BindingTaskCreator storecontract.BindingTaskCreator
@@ -569,6 +572,16 @@ func (s *server) DeleteMapping(ctx context.Context, r *pb.DeleteMappingRequest)
return &pb.DeleteMappingResponse{}, nil
}
func (s *server) RecordMappingMatch(ctx context.Context, r *pb.RecordMappingMatchRequest) (*pb.RecordMappingMatchResponse, error) {
if s.deps.MappingMatches == nil {
return nil, status.Error(codes.Unavailable, "mapping match recorder unavailable")
}
if err := s.deps.MappingMatches.RecordMappingMatch(ctx, r.MappingId, r.CaptureId); err != nil {
return nil, s.logErr("RecordMappingMatch", err)
}
return &pb.RecordMappingMatchResponse{}, nil
}
// Binding
func (s *server) binding() (storecontract.BindingStore, error) {
+8 -6
View File
@@ -178,8 +178,9 @@ func mappingFieldValue(f *pb.MappingField) mapping.Field {
func mappingProto(m mapping.Mapping) *pb.Mapping {
return &pb.Mapping{
Id: m.ID, Name: m.Name, SourceHint: m.SourceHint,
Fields: mapValues(m.Fields, mappingFieldProto),
Id: m.ID, Name: m.Name, SourceHint: m.SourceHint, EventTypeId: m.EventTypeID,
Fields: mapValues(m.Fields, mappingFieldProto),
MatchCount: m.MatchCount, LastMatchedAt: timestamp(m.LastMatchedAt),
CreatedAt: timestamp(m.CreatedAt), UpdatedAt: timestamp(m.UpdatedAt),
}
}
@@ -189,8 +190,9 @@ func mappingValue(m *pb.Mapping) mapping.Mapping {
return mapping.Mapping{}
}
return mapping.Mapping{
ID: m.Id, Name: m.Name, SourceHint: m.SourceHint,
Fields: mapValues(m.Fields, mappingFieldValue),
ID: m.Id, Name: m.Name, SourceHint: m.SourceHint, EventTypeID: m.EventTypeId,
Fields: mapValues(m.Fields, mappingFieldValue),
MatchCount: m.MatchCount, LastMatchedAt: timeValue(m.LastMatchedAt),
CreatedAt: timeValue(m.CreatedAt), UpdatedAt: timeValue(m.UpdatedAt),
}
}
@@ -198,7 +200,7 @@ func mappingValue(m *pb.Mapping) mapping.Mapping {
func bindingProto(b binding.Binding) *pb.Binding {
return &pb.Binding{
Id: b.ID, Name: b.Name, Matcher: &pb.BindingMatcher{Source: b.Matcher.Source},
MappingId: b.MappingID, Workflow: b.Workflow, Owner: b.Owner, Repo: b.Repo,
MappingId: b.MappingID, Filter: b.Filter, Workflow: b.Workflow, Owner: b.Owner, Repo: b.Repo,
Version: int64(b.Version), Status: string(b.Status), Secret: b.Secret,
CreatedAt: timestamp(b.CreatedAt), UpdatedAt: timestamp(b.UpdatedAt),
}
@@ -214,7 +216,7 @@ func bindingValue(b *pb.Binding) binding.Binding {
}
return binding.Binding{
ID: b.Id, Name: b.Name, Matcher: binding.Matcher{Source: source},
MappingID: b.MappingId, Workflow: b.Workflow, Owner: b.Owner, Repo: b.Repo,
MappingID: b.MappingId, Filter: b.Filter, Workflow: b.Workflow, Owner: b.Owner, Repo: b.Repo,
Version: int(b.Version), Status: binding.Status(b.Status), Secret: b.Secret,
CreatedAt: timeValue(b.CreatedAt), UpdatedAt: timeValue(b.UpdatedAt),
}
+3 -5
View File
@@ -398,11 +398,9 @@ func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, w
// binding_version on the new row in a second statement.
//
// A crash between the two writes leaves the task without provenance
// (binding_id == 0); the binding_dispatches ledger row written by the
// dispatch loop still records the (binding, capture, task) triple, so
// a future repair pass could backfill. This is the same
// best-effort-provenance pattern the rest of the task lifecycle uses
// for fields added after the row's primary insert.
// (binding_id == 0). This is the same best-effort-provenance pattern the
// rest of the task lifecycle uses for fields added after the row's primary
// insert.
func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int) (*workflow.Task, error) {
t, err := s.EnqueueChatTask(ctx, owner, repo, title, body, wf, identity)
if err != nil {
+33 -8
View File
@@ -16,6 +16,7 @@ type bindingRequest struct {
Name string `json:"name"`
Matcher binding.Matcher `json:"matcher"`
MappingID string `json:"mapping_id"`
Filter string `json:"filter"`
Workflow string `json:"workflow"`
// Owner and Repo optionally pin the binding to one configured repo,
// for multi-repo deployments. Both empty is valid (falls back to the
@@ -67,6 +68,7 @@ func (s *Server) handleBindingCreate(w http.ResponseWriter, r *http.Request) {
Name: req.Name,
Matcher: req.Matcher,
MappingID: req.MappingID,
Filter: req.Filter,
Workflow: req.Workflow,
Owner: req.Owner,
Repo: req.Repo,
@@ -77,12 +79,11 @@ func (s *Server) handleBindingCreate(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !s.checkBindingFilter(w, r, b) {
return
}
id, err := s.Bindings.InsertBinding(r.Context(), b)
if err != nil {
if errors.Is(err, storecontract.ErrBindingOverlap) {
http.Error(w, "binding source overlaps an existing binding", http.StatusConflict)
return
}
s.Log.Error("insert binding", "err", err)
http.Error(w, "create binding failed", http.StatusInternalServerError)
return
@@ -154,6 +155,7 @@ func (s *Server) handleBindingUpdate(w http.ResponseWriter, r *http.Request) {
Name: req.Name,
Matcher: req.Matcher,
MappingID: req.MappingID,
Filter: req.Filter,
Workflow: req.Workflow,
Owner: req.Owner,
Repo: req.Repo,
@@ -163,12 +165,13 @@ func (s *Server) handleBindingUpdate(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !s.checkBindingFilter(w, r, b) {
return
}
if err := s.Bindings.UpdateBinding(r.Context(), b); err != nil {
switch {
case errors.Is(err, storecontract.ErrBindingNotFound):
http.Error(w, "binding not found", http.StatusNotFound)
case errors.Is(err, storecontract.ErrBindingOverlap):
http.Error(w, "binding source overlaps an existing binding", http.StatusConflict)
default:
s.Log.Error("update binding", "err", err, "id", id)
http.Error(w, "update binding failed", http.StatusInternalServerError)
@@ -229,8 +232,6 @@ func (s *Server) handleBindingApprove(w http.ResponseWriter, r *http.Request) {
http.Error(w, "binding not found", http.StatusNotFound)
case errors.Is(err, storecontract.ErrBindingTransition):
http.Error(w, "binding cannot be approved from its current state", http.StatusConflict)
case errors.Is(err, storecontract.ErrBindingOverlap):
http.Error(w, "binding source overlaps an existing binding", http.StatusConflict)
default:
s.Log.Error("approve binding", "err", err, "id", id)
http.Error(w, "approve binding failed", http.StatusInternalServerError)
@@ -247,6 +248,30 @@ func (s *Server) handleBindingApprove(w http.ResponseWriter, r *http.Request) {
writeJSON(w, updated)
}
// checkBindingFilter refuses a binding whose mapping does not exist or whose
// filter does not compile against the mapping's parameters.
func (s *Server) checkBindingFilter(w http.ResponseWriter, r *http.Request, b binding.Binding) bool {
if s.Mappings == nil {
http.Error(w, "mappings not configured", http.StatusServiceUnavailable)
return false
}
m, err := s.Mappings.GetMapping(r.Context(), b.MappingID)
if err != nil {
s.Log.Error("get mapping for binding", "err", err, "mapping", b.MappingID)
http.Error(w, "get mapping failed", http.StatusInternalServerError)
return false
}
if m == nil {
http.Error(w, "binding: mapping not found: "+b.MappingID, http.StatusBadRequest)
return false
}
if _, err := binding.CompileFilter(b.Filter, m.Fields); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return false
}
return true
}
// stripBindingSecrets zeroes the Secret field on every binding in the slice
// so the list handler never leaks a shared HMAC secret through /api/bindings.
// Single-binding reads do the same inline; this covers the list path.
+137
View File
@@ -0,0 +1,137 @@
package webui
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"testing"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
)
// memBindings keeps mappings and bindings in memory with no per-source limit,
// as the Postgres store does.
type memBindings struct {
mappings map[string]mapping.Mapping
bindings map[string]binding.Binding
}
func newMemBindings() *memBindings {
return &memBindings{mappings: map[string]mapping.Mapping{}, bindings: map[string]binding.Binding{}}
}
func (m *memBindings) InsertMapping(_ context.Context, v mapping.Mapping) (string, error) {
v.ID = fmt.Sprintf("m%d", len(m.mappings)+1)
m.mappings[v.ID] = v
return v.ID, nil
}
func (m *memBindings) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) {
v, ok := m.mappings[id]
if !ok {
return nil, nil
}
return &v, nil
}
func (m *memBindings) ListMappings(context.Context) ([]mapping.Mapping, error) { return nil, nil }
func (m *memBindings) UpdateMapping(context.Context, mapping.Mapping) error { return nil }
func (m *memBindings) DeleteMapping(context.Context, string) error { return nil }
func (m *memBindings) InsertBinding(_ context.Context, b binding.Binding) (string, error) {
b.ID = fmt.Sprintf("b%d", len(m.bindings)+1)
m.bindings[b.ID] = b
return b.ID, nil
}
func (m *memBindings) GetBinding(_ context.Context, id string) (*binding.Binding, error) {
v, ok := m.bindings[id]
if !ok {
return nil, nil
}
return &v, nil
}
func (m *memBindings) ListBindings(context.Context) ([]binding.Binding, error) { return nil, nil }
func (m *memBindings) UpdateBinding(_ context.Context, b binding.Binding) error {
if _, ok := m.bindings[b.ID]; !ok {
return storecontract.ErrBindingNotFound
}
m.bindings[b.ID] = b
return nil
}
func (m *memBindings) DeleteBinding(context.Context, string) error { return nil }
func (m *memBindings) ApproveBinding(context.Context, string) error { return nil }
func memBindingServer(t *testing.T) (*Server, string) {
t.Helper()
store := newMemBindings()
mappingID, _ := store.InsertMapping(t.Context(), mapping.Mapping{
Name: "m", EventTypeID: "et-1",
Fields: []mapping.Field{{Name: "title", Path: "issue.title", Type: mapping.TypeString}},
})
return &Server{
Log: slog.New(slog.DiscardHandler),
Mappings: store,
Bindings: store,
ControlPlane: workflowControlPlane(t, workflow.WorkflowDefinitionCollection{Definitions: []workflow.WorkflowDefinitionEntry{{ID: "implement", YAML: "id: implement\nsteps:\n - type: implement.prepare\n"}}}),
}, mappingID
}
// Several bindings may share a source.
func TestHandleBindingCreateAllowsSeveralPerSource(t *testing.T) {
srv, mappingID := memBindingServer(t)
for _, suffix := range []string{"first", "second"} {
w := doJSON(t, srv, http.MethodPost, "/api/bindings", validBindingRequest(suffix, "sentry", mappingID))
if w.Code != http.StatusCreated {
t.Fatalf("%s create status = %d, want %d; body = %s", suffix, w.Code, http.StatusCreated, w.Body.String())
}
}
}
// A binding's filter is checked against its mapping's parameters on save.
func TestHandleBindingFilterIsCheckedAgainstMapping(t *testing.T) {
tests := []struct {
name string
filter string
want int
}{
{"no filter", "", http.StatusCreated},
{"filter on a mapped parameter", `title.startsWith("P1")`, http.StatusCreated},
{"unknown parameter", `severity == "high"`, http.StatusBadRequest},
{"not boolean", `title`, http.StatusBadRequest},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv, mappingID := memBindingServer(t)
req := validBindingRequest("f", "sentry", mappingID)
req["filter"] = tt.filter
w := doJSON(t, srv, http.MethodPost, "/api/bindings", req)
if w.Code != tt.want {
t.Fatalf("create status = %d, want %d; body = %s", w.Code, tt.want, w.Body.String())
}
if w.Code == http.StatusCreated {
var created binding.Binding
_ = json.Unmarshal(w.Body.Bytes(), &created)
if created.Filter != tt.filter {
t.Fatalf("created.Filter = %q, want %q", created.Filter, tt.filter)
}
w = doJSON(t, srv, http.MethodPatch, "/api/bindings/"+created.ID, req)
if w.Code != http.StatusOK {
t.Fatalf("update status = %d; body = %s", w.Code, w.Body.String())
}
}
})
}
}
func TestHandleBindingRejectsUnknownMapping(t *testing.T) {
srv, _ := memBindingServer(t)
w := doJSON(t, srv, http.MethodPost, "/api/bindings", validBindingRequest("x", "sentry", "absent"))
if w.Code != http.StatusBadRequest {
t.Fatalf("create status = %d, want %d; body = %s", w.Code, http.StatusBadRequest, w.Body.String())
}
}
-17
View File
@@ -187,23 +187,6 @@ func TestHandleBindingsListStripsSecrets(t *testing.T) {
}
}
func TestHandleBindingCreateRejectsOverlap(t *testing.T) {
srv := bindingTestServer(t)
mappingID := seedMapping(t, srv, "m")
w := doJSON(t, srv, http.MethodPost, "/api/bindings", validBindingRequest("first", "sentry", mappingID))
if w.Code != http.StatusCreated {
t.Fatalf("first create status = %d, want %d; body = %s", w.Code, http.StatusCreated, w.Body.String())
}
// Overlap because Per bead t2db.4, two bindings may not own the same
// source; the second insert must surface ErrBindingOverlap -> 409.
w = doJSON(t, srv, http.MethodPost, "/api/bindings", validBindingRequest("second", "sentry", mappingID))
if w.Code != http.StatusConflict {
t.Fatalf("second create status = %d, want %d; body = %s", w.Code, http.StatusConflict, w.Body.String())
}
}
func TestHandleBindingCreateRejectsMissingWorkflow(t *testing.T) {
srv := bindingTestServer(t)
mappingID := seedMapping(t, srv, "m")
+38 -9
View File
@@ -14,9 +14,40 @@ import (
// PATCH /api/mappings/{id} both accept. ID/CreatedAt/UpdatedAt are
// server-assigned, never taken from the request body.
type mappingRequest struct {
Name string `json:"name"`
SourceHint string `json:"source_hint"`
Fields []mapping.Field `json:"fields"`
Name string `json:"name"`
SourceHint string `json:"source_hint"`
EventTypeID string `json:"event_type_id"`
Fields []mapping.Field `json:"fields"`
}
// validateMapping checks m is well-formed and fits its event type's schema.
// It writes the refusal and returns false when it does not.
func (s *Server) validateMapping(w http.ResponseWriter, r *http.Request, m mapping.Mapping) bool {
if err := m.Validate(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return false
}
if s.EventTypes == nil {
http.Error(w, "event types not configured", http.StatusServiceUnavailable)
return false
}
types, err := s.EventTypes.ListEventTypes(r.Context())
if err != nil {
s.Log.Error("list event types", "err", err)
http.Error(w, "list event types failed", http.StatusInternalServerError)
return false
}
for _, t := range types {
if t.ID == m.EventTypeID {
if err := mapping.CheckSchema(m.Fields, t.Schema); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return false
}
return true
}
}
http.Error(w, "mapping: event type not found: "+m.EventTypeID, http.StatusBadRequest)
return false
}
func (s *Server) handleMappingsList(w http.ResponseWriter, r *http.Request) {
@@ -46,9 +77,8 @@ func (s *Server) handleMappingCreate(w http.ResponseWriter, r *http.Request) {
http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
return
}
m := mapping.Mapping{Name: req.Name, SourceHint: req.SourceHint, Fields: req.Fields}
if err := m.Validate(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
m := mapping.Mapping{Name: req.Name, SourceHint: req.SourceHint, EventTypeID: req.EventTypeID, Fields: req.Fields}
if !s.validateMapping(w, r, m) {
return
}
id, err := s.Mappings.InsertMapping(r.Context(), m)
@@ -108,9 +138,8 @@ func (s *Server) handleMappingUpdate(w http.ResponseWriter, r *http.Request) {
http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
return
}
m := mapping.Mapping{ID: id, Name: req.Name, SourceHint: req.SourceHint, Fields: req.Fields}
if err := m.Validate(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
m := mapping.Mapping{ID: id, Name: req.Name, SourceHint: req.SourceHint, EventTypeID: req.EventTypeID, Fields: req.Fields}
if !s.validateMapping(w, r, m) {
return
}
if err := s.Mappings.UpdateMapping(r.Context(), m); err != nil {
+82 -18
View File
@@ -9,6 +9,7 @@ import (
"path/filepath"
"testing"
"github.com/samcharles93/archie-core/internal/domain/eventtype"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/infrastructure/edastore"
"github.com/samcharles93/archie-core/internal/store"
@@ -26,10 +27,66 @@ func mappingTestServer(t *testing.T) *Server {
t.Cleanup(func() { _ = s.Close() })
eda := edastore.OpenTest(t)
return &Server{
Store: s,
Log: slog.New(slog.DiscardHandler),
Mappings: eda,
Captures: eda,
Store: s,
Log: slog.New(slog.DiscardHandler),
Mappings: eda,
Captures: eda,
EventTypes: testEventTypes(),
}
}
// testEventTypes holds one event type, et-1, whose schema has every path the
// mapping tests write.
func testEventTypes() *memEventTypes {
return &memEventTypes{types: []eventtype.EventType{{
ID: "et-1", Source: "sentry", Name: "issue",
Schema: map[string]eventtype.ValueType{
"a": eventtype.TypeString, "b": eventtype.TypeBool,
"issue": eventtype.TypeObject, "issue.title": eventtype.TypeString,
},
}}}
}
// A mapping belongs to an event type and is checked against its schema.
func TestHandleMappingSaveChecksEventTypeSchema(t *testing.T) {
tests := []struct {
name string
eventType string
path string
fieldType mapping.FieldType
want int
}{
{"fits the schema", "et-1", "issue.title", mapping.TypeString, http.StatusCreated},
{"no event type", "", "issue.title", mapping.TypeString, http.StatusBadRequest},
{"unknown event type", "et-9", "issue.title", mapping.TypeString, http.StatusBadRequest},
{"path the type does not have", "et-1", "sender.login", mapping.TypeString, http.StatusBadRequest},
{"type differs from the schema", "et-1", "issue.title", mapping.TypeNumber, http.StatusBadRequest},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := mappingTestServer(t)
req := map[string]any{
"name": "m",
"event_type_id": tt.eventType,
"fields": []mapping.Field{{Name: "f", Path: tt.path, Type: tt.fieldType}},
}
if w := doJSON(t, srv, http.MethodPost, "/api/mappings", req); w.Code != tt.want {
t.Fatalf("create status = %d, want %d; body = %s", w.Code, tt.want, w.Body.String())
}
ok := doJSON(t, srv, http.MethodPost, "/api/mappings", map[string]any{
"name": "base", "event_type_id": "et-1",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
})
var created mapping.Mapping
_ = json.Unmarshal(ok.Body.Bytes(), &created)
want := tt.want
if want == http.StatusCreated {
want = http.StatusOK
}
if w := doJSON(t, srv, http.MethodPatch, "/api/mappings/"+created.ID, req); w.Code != want {
t.Fatalf("update status = %d, want %d; body = %s", w.Code, want, w.Body.String())
}
})
}
}
@@ -74,8 +131,9 @@ func doJSONWithHeaders(t *testing.T, srv *Server, method, path string, body any,
func TestHandleMappingCreateAndGet(t *testing.T) {
srv := mappingTestServer(t)
w := doJSON(t, srv, http.MethodPost, "/api/mappings", map[string]any{
"name": "sentry issue opened",
"source_hint": "sentry",
"name": "sentry issue opened",
"source_hint": "sentry",
"event_type_id": "et-1",
"fields": []mapping.Field{
{Name: "title", Path: "issue.title", Type: mapping.TypeString, Required: true},
},
@@ -124,8 +182,9 @@ func TestHandleMappingsList(t *testing.T) {
srv := mappingTestServer(t)
for range 3 {
doJSON(t, srv, http.MethodPost, "/api/mappings", map[string]any{
"name": "m",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
"name": "m",
"event_type_id": "et-1",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
})
}
w := doJSON(t, srv, http.MethodGet, "/api/mappings", nil)
@@ -146,15 +205,17 @@ func TestHandleMappingsList(t *testing.T) {
func TestHandleMappingUpdate(t *testing.T) {
srv := mappingTestServer(t)
w := doJSON(t, srv, http.MethodPost, "/api/mappings", map[string]any{
"name": "orig",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
"name": "orig",
"event_type_id": "et-1",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
})
var created mapping.Mapping
_ = json.Unmarshal(w.Body.Bytes(), &created)
w = doJSON(t, srv, http.MethodPatch, "/api/mappings/"+created.ID, map[string]any{
"name": "renamed",
"fields": []mapping.Field{{Name: "b", Path: "b", Type: mapping.TypeBool}},
"name": "renamed",
"event_type_id": "et-1",
"fields": []mapping.Field{{Name: "b", Path: "b", Type: mapping.TypeBool}},
})
if w.Code != http.StatusOK {
t.Fatalf("update status = %d, want %d; body = %s", w.Code, http.StatusOK, w.Body.String())
@@ -171,8 +232,9 @@ func TestHandleMappingUpdate(t *testing.T) {
func TestHandleMappingUpdateMissingReturns404(t *testing.T) {
srv := mappingTestServer(t)
w := doJSON(t, srv, http.MethodPatch, "/api/mappings/999", map[string]any{
"name": "x",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
"name": "x",
"event_type_id": "et-1",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
})
if w.Code != http.StatusNotFound {
t.Fatalf("status = %d, want %d; body = %s", w.Code, http.StatusNotFound, w.Body.String())
@@ -182,8 +244,9 @@ func TestHandleMappingUpdateMissingReturns404(t *testing.T) {
func TestHandleMappingDelete(t *testing.T) {
srv := mappingTestServer(t)
w := doJSON(t, srv, http.MethodPost, "/api/mappings", map[string]any{
"name": "m",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
"name": "m",
"event_type_id": "et-1",
"fields": []mapping.Field{{Name: "a", Path: "a", Type: mapping.TypeString}},
})
var created mapping.Mapping
_ = json.Unmarshal(w.Body.Bytes(), &created)
@@ -274,8 +337,9 @@ func TestHandleMappingsUnavailableWhenNotConfigured(t *testing.T) {
func TestHandleMappingMutationsRequireCSRFHeader(t *testing.T) {
srv := mappingTestServer(t)
newMapping := map[string]any{
"name": "sentry issue opened",
"source_hint": "sentry",
"name": "sentry issue opened",
"source_hint": "sentry",
"event_type_id": "et-1",
"fields": []mapping.Field{
{Name: "title", Path: "issue.title", Type: mapping.TypeString, Required: true},
},
+14
View File
@@ -119,6 +119,9 @@ service StateStoreService {
rpc ListMappings(ListMappingsRequest) returns (ListMappingsResponse);
rpc UpdateMapping(UpdateMappingRequest) returns (UpdateMappingResponse);
rpc DeleteMapping(DeleteMappingRequest) returns (DeleteMappingResponse);
// RecordMappingMatch counts one event a mapping resolved; the same
// (mapping, capture) twice counts once.
rpc RecordMappingMatch(RecordMappingMatchRequest) returns (RecordMappingMatchResponse);
// EventType (docs/prds/event-automation.md, "Event types"). Insert and
// Update refuse an overlapping or malformed type.
@@ -362,6 +365,10 @@ message Mapping {
repeated MappingField fields = 4;
google.protobuf.Timestamp created_at = 5;
google.protobuf.Timestamp updated_at = 6;
string event_type_id = 7;
// match_count and last_matched_at are read-only: the store counts them.
int64 match_count = 8;
google.protobuf.Timestamp last_matched_at = 9;
}
// BindingMatcher mirrors internal/domain/binding.Matcher.
@@ -383,6 +390,7 @@ message Binding {
string secret = 10;
google.protobuf.Timestamp created_at = 11;
google.protobuf.Timestamp updated_at = 12;
string filter = 13;
}
// WorkflowStat mirrors internal/store.WorkflowStat.
@@ -724,6 +732,12 @@ message UpdateMappingResponse {}
message DeleteMappingRequest { string id = 1; }
message DeleteMappingResponse {}
message RecordMappingMatchRequest {
string mapping_id = 1;
string capture_id = 2;
}
message RecordMappingMatchResponse {}
message InsertEventTypeRequest { EventType event_type = 1; }
message InsertEventTypeResponse { string id = 1; }