From 168247d9ccc2c35e128d79ba8838c36ffc1b4202 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 8 Oct 2026 21:02:21 +1100 Subject: [PATCH] fix(capture): inherit source org on ingestion --- .../postgres/dispatch_org_test.go | 114 +++++++++++++++--- internal/infrastructure/postgres/edastore.go | 24 ++-- .../postgres/postgresdb/edastore.sql.go | 32 +++-- .../postgres/queries/edastore.sql | 6 +- 4 files changed, 133 insertions(+), 43 deletions(-) diff --git a/internal/infrastructure/postgres/dispatch_org_test.go b/internal/infrastructure/postgres/dispatch_org_test.go index 4b9f24e2c..57a226548 100644 --- a/internal/infrastructure/postgres/dispatch_org_test.go +++ b/internal/infrastructure/postgres/dispatch_org_test.go @@ -1,17 +1,26 @@ package postgres_test import ( + "net" + "net/http" + "net/http/httptest" + "strings" "testing" "time" + "google.golang.org/grpc" + "github.com/samcharles93/archie-core/internal/domain/binding" "github.com/samcharles93/archie-core/internal/domain/eventtype" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/mapping" "github.com/samcharles93/archie-core/internal/domain/org" + "github.com/samcharles93/archie-core/internal/domain/source" "github.com/samcharles93/archie-core/internal/domain/storecontract" + "github.com/samcharles93/archie-core/internal/infrastructure/captureintake" "github.com/samcharles93/archie-core/internal/infrastructure/postgres" "github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore" + "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" ) func TestPendingDispatchInCallerOrg(t *testing.T) { @@ -35,9 +44,23 @@ func TestPendingDispatchInCallerOrg(t *testing.T) { t.Fatal(err) } eda := postgres.NewEDA(db.Pool, nil) + client := captureRPC(t, eda, db) + receiver := &captureintake.Receiver{Captures: client, Sources: client, Retention: time.Hour, MaxEvents: 100} + mux := http.NewServeMux() + receiver.Register(mux) ids := map[org.OrgID]string{} - for n, owner := range []org.OrgID{org.DefaultOrgID, "acme"} { - typeID, err := eda.InsertEventType(ctx, eventtype.EventType{Source: "webhook", Name: string(owner), Rule: eventtype.Rule{Headers: []eventtype.HeaderCondition{{Name: "X-Org", Value: string(owner)}}}}) + bindings := map[org.OrgID]string{} + paths := []string{} + for _, owner := range []org.OrgID{org.DefaultOrgID, "acme"} { + path := "webhook-" + string(owner) + paths = append(paths, path) + if err := eda.InsertSource(ctx, source.Source{Path: path, Signing: source.SigningUnsigned}); err != nil { + t.Fatal(err) + } + if _, err := db.Pool.Exec(ctx, "UPDATE sources SET org_id=$1, workspace_id='incoming' WHERE path=$2", owner, path); err != nil { + t.Fatal(err) + } + typeID, err := eda.InsertEventType(ctx, eventtype.EventType{Source: path, Name: string(owner), Rule: eventtype.Rule{Headers: []eventtype.HeaderCondition{{Name: "X-Org", Value: string(owner)}}}}) if err != nil { t.Fatal(err) } @@ -45,38 +68,95 @@ func TestPendingDispatchInCallerOrg(t *testing.T) { if err != nil { t.Fatal(err) } - bindingID, err := eda.InsertBinding(ctx, binding.Binding{Name: string(owner), Matcher: binding.Matcher{Source: "webhook"}, MappingID: mappingID, Workflow: "deploy"}) + bindingID, err := eda.InsertBinding(ctx, binding.Binding{Name: string(owner), Matcher: binding.Matcher{Source: path}, MappingID: mappingID, Workflow: "deploy"}) if err != nil { t.Fatal(err) } if err := eda.ApproveBinding(ctx, bindingID); err != nil { t.Fatal(err) } - captureID, err := eda.InsertCapture(ctx, storecontract.CapturedEvent{Source: "webhook", Body: "{}", Headers: `{"X-Org":["` + string(owner) + `"]}`, Authenticated: true, EventType: typeID, ReceivedAt: time.Now().Add(time.Duration(n) * time.Minute)}, time.Hour, 100) - if err != nil { - t.Fatal(err) - } - // EDA writes do not yet take org ownership; model their persisted org - // here to exercise the dispatch read independently of that work. - for _, record := range []struct{ table, id string }{{"event_types", typeID}, {"mappings", mappingID}, {"bindings", bindingID}, {"captures", captureID}} { + // Catalogue writes still default to the system org; capture ownership + // must come from the persisted source, never a caller-supplied org. + for _, record := range []struct{ table, id string }{{"event_types", typeID}, {"mappings", mappingID}, {"bindings", bindingID}} { if _, err := db.Pool.Exec(ctx, "UPDATE "+record.table+" SET org_id=$1 WHERE id=$2", owner, record.id); err != nil { t.Fatal(err) } } - ids[owner] = captureID - if owner == "acme" { - run, err := db.EnqueueBindingTask(org.WithOrg(ctx, owner), "", "", "Org dispatch", "", "deploy", "acme-bot", bindingID, 1, nil) - if err != nil || run.Org != owner || run.Identity != "acme-bot" { - t.Fatalf("binding task %#v err %v", run, err) - } + request := httptest.NewRequestWithContext(ctx, http.MethodPost, "/webhooks/capture/"+path, strings.NewReader("{}")) + request.Header.Set("X-Org", string(owner)) + response := httptest.NewRecorder() + mux.ServeHTTP(response, request) + if response.Code != http.StatusAccepted { + t.Fatalf("capture: %d %s", response.Code, response.Body.String()) } + var captureID, captureOrg, workspace string + if err := db.Pool.QueryRow(ctx, "SELECT id, org_id, workspace_id FROM captures WHERE source=$1", path).Scan(&captureID, &captureOrg, &workspace); err != nil { + t.Fatal(err) + } + if captureOrg != string(owner) || workspace != "incoming" { + t.Fatalf("capture org %s workspace %s, want %s/incoming", captureOrg, workspace, owner) + } + ids[owner] = captureID + bindings[owner] = bindingID } for _, owner := range []org.OrgID{org.DefaultOrgID, "acme"} { t.Run(string(owner), func(t *testing.T) { - captures, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), []string{"webhook"}, 1) + captures, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), paths, 1) if err != nil || len(captures) != 1 || captures[0].ID != ids[owner] { t.Fatalf("org %s got captures %#v err %v", owner, captures, err) } + bindingID := bindings[owner] + if err := client.RecordDispatch(t.Context(), bindingID, 1, ids[owner], 0, ""); err != nil { + t.Fatal(err) + } + taskIdentity := "" + if owner == "acme" { + taskIdentity = "acme-bot" + } + run, err := client.EnqueueBindingTask(t.Context(), "", "", "Org dispatch", "", "deploy", taskIdentity, bindingID, 1, nil) + if err != nil || run.Org != owner { + t.Fatalf("binding task %#v err %v", run, err) + } + if err := client.SetDispatchTask(t.Context(), bindingID, ids[owner], run.ID); err != nil { + t.Fatal(err) + } + pending, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), paths, 1) + if err != nil || len(pending) != 0 { + t.Fatalf("dispatched capture still pending: %#v err %v", pending, err) + } }) } + t.Run("unknown source", func(t *testing.T) { + id, err := client.InsertCapture(org.WithOrg(t.Context(), "acme"), storecontract.CapturedEvent{Source: "unknown", Body: "{}"}, time.Hour, 100) + if err != nil { + t.Fatal(err) + } + var owner, workspace string + if err := db.Pool.QueryRow(t.Context(), "SELECT org_id,workspace_id FROM captures WHERE id=$1", id).Scan(&owner, &workspace); err != nil { + t.Fatal(err) + } + if owner != string(org.DefaultOrgID) || workspace != string(org.DefaultWorkspaceID) { + t.Fatalf("unknown source owner %s/%s", owner, workspace) + } + }) +} + +func captureRPC(t *testing.T, eda *postgres.EDA, db *pgstore.TaskDB) *staterpc.Client { + t.Helper() + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + grants := &staterpc.TaskGrants{} + srv := grpc.NewServer(grpc.ChainUnaryInterceptor(grants.UnaryInterceptor("test-token")), grpc.ChainStreamInterceptor(grants.StreamInterceptor("test-token"))) + staterpc.RegisterServer(srv, staterpc.Deps{Tasks: db, Captures: eda, Sources: eda, BindingDispatcher: eda, BindingTaskCreator: db}) + t.Cleanup(srv.Stop) + go func() { _ = srv.Serve(listener) }() + client, closeClient, err := staterpc.Dial(listener.Addr().String(), "archied", "test-token") + if err != nil { + t.Fatal(err) + } + t.Cleanup(closeClient) + return client } diff --git a/internal/infrastructure/postgres/edastore.go b/internal/infrastructure/postgres/edastore.go index 9ac847119..c5339fb23 100644 --- a/internal/infrastructure/postgres/edastore.go +++ b/internal/infrastructure/postgres/edastore.go @@ -91,17 +91,19 @@ func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent, delivery = id } if err := s.q.InsertCapture(ctx, postgresdb.InsertCaptureParams{ - ID: id, - Source: c.Source, - RemoteAddr: c.RemoteAddr, - ContentType: c.ContentType, - Headers: c.Headers, - Body: c.Body, - Authenticated: c.Authenticated, - Unsigned: c.Unsigned, - ReceivedAt: received.UTC(), - EventType: eventType, - Delivery: delivery, + ID: id, + Source: c.Source, + RemoteAddr: c.RemoteAddr, + ContentType: c.ContentType, + Headers: c.Headers, + Body: c.Body, + Authenticated: c.Authenticated, + Unsigned: c.Unsigned, + ReceivedAt: received.UTC(), + EventType: eventType, + Delivery: delivery, + DefaultOrg: string(org.DefaultOrgID), + DefaultWorkspace: string(org.DefaultWorkspaceID), }); err != nil { return "", fmt.Errorf("edastore: insert capture: %w", err) } diff --git a/internal/infrastructure/postgres/postgresdb/edastore.sql.go b/internal/infrastructure/postgres/postgresdb/edastore.sql.go index 85b143e40..cfb67e0ec 100644 --- a/internal/infrastructure/postgres/postgresdb/edastore.sql.go +++ b/internal/infrastructure/postgres/postgresdb/edastore.sql.go @@ -443,22 +443,26 @@ func (q *Queries) InsertBindingDispatch(ctx context.Context, arg InsertBindingDi const insertCapture = `-- name: InsertCapture :exec -INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) +INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery, org_id, workspace_id) +SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, + COALESCE(s.org_id, $12::text), COALESCE(s.workspace_id, $13::text) +FROM (SELECT 1) intake LEFT JOIN sources s ON s.path = $2 ` type InsertCaptureParams struct { - ID string - Source string - RemoteAddr string - ContentType string - Headers string - Body string - Authenticated bool - ReceivedAt time.Time - Unsigned bool - EventType string - Delivery string + ID string + Source string + RemoteAddr string + ContentType string + Headers string + Body string + Authenticated bool + ReceivedAt time.Time + Unsigned bool + EventType string + Delivery string + DefaultOrg string + DefaultWorkspace string } // EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls. @@ -475,6 +479,8 @@ func (q *Queries) InsertCapture(ctx context.Context, arg InsertCaptureParams) er arg.Unsigned, arg.EventType, arg.Delivery, + arg.DefaultOrg, + arg.DefaultWorkspace, ) return err } diff --git a/internal/infrastructure/postgres/queries/edastore.sql b/internal/infrastructure/postgres/queries/edastore.sql index 35e2fb400..1a09188b1 100644 --- a/internal/infrastructure/postgres/queries/edastore.sql +++ b/internal/infrastructure/postgres/queries/edastore.sql @@ -1,8 +1,10 @@ -- EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls. -- name: InsertCapture :exec -INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11); +INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery, org_id, workspace_id) +SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, + COALESCE(s.org_id, @default_org::text), COALESCE(s.workspace_id, @default_workspace::text) +FROM (SELECT 1) intake LEFT JOIN sources s ON s.path = $2; -- name: ListCaptures :many SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, org_id, workspace_id, delivery