fix(capture): inherit source org on ingestion
This commit is contained in:
4 files changed
+133
-43
No files matched your search
@@ -1,17 +1,26 @@
|
||||
package postgres_test
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/binding"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eventtype"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/mapping"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/source"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/captureintake"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/postgres/pgstore"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
)
|
||||
|
||||
func TestPendingDispatchInCallerOrg(t *testing.T) {
|
||||
@@ -35,9 +44,23 @@ func TestPendingDispatchInCallerOrg(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eda := postgres.NewEDA(db.Pool, nil)
|
||||
client := captureRPC(t, eda, db)
|
||||
receiver := &captureintake.Receiver{Captures: client, Sources: client, Retention: time.Hour, MaxEvents: 100}
|
||||
mux := http.NewServeMux()
|
||||
receiver.Register(mux)
|
||||
ids := map[org.OrgID]string{}
|
||||
for n, owner := range []org.OrgID{org.DefaultOrgID, "acme"} {
|
||||
typeID, err := eda.InsertEventType(ctx, eventtype.EventType{Source: "webhook", Name: string(owner), Rule: eventtype.Rule{Headers: []eventtype.HeaderCondition{{Name: "X-Org", Value: string(owner)}}}})
|
||||
bindings := map[org.OrgID]string{}
|
||||
paths := []string{}
|
||||
for _, owner := range []org.OrgID{org.DefaultOrgID, "acme"} {
|
||||
path := "webhook-" + string(owner)
|
||||
paths = append(paths, path)
|
||||
if err := eda.InsertSource(ctx, source.Source{Path: path, Signing: source.SigningUnsigned}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.Pool.Exec(ctx, "UPDATE sources SET org_id=$1, workspace_id='incoming' WHERE path=$2", owner, path); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
typeID, err := eda.InsertEventType(ctx, eventtype.EventType{Source: path, Name: string(owner), Rule: eventtype.Rule{Headers: []eventtype.HeaderCondition{{Name: "X-Org", Value: string(owner)}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -45,38 +68,95 @@ func TestPendingDispatchInCallerOrg(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bindingID, err := eda.InsertBinding(ctx, binding.Binding{Name: string(owner), Matcher: binding.Matcher{Source: "webhook"}, MappingID: mappingID, Workflow: "deploy"})
|
||||
bindingID, err := eda.InsertBinding(ctx, binding.Binding{Name: string(owner), Matcher: binding.Matcher{Source: path}, MappingID: mappingID, Workflow: "deploy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := eda.ApproveBinding(ctx, bindingID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
captureID, err := eda.InsertCapture(ctx, storecontract.CapturedEvent{Source: "webhook", Body: "{}", Headers: `{"X-Org":["` + string(owner) + `"]}`, Authenticated: true, EventType: typeID, ReceivedAt: time.Now().Add(time.Duration(n) * time.Minute)}, time.Hour, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// EDA writes do not yet take org ownership; model their persisted org
|
||||
// here to exercise the dispatch read independently of that work.
|
||||
for _, record := range []struct{ table, id string }{{"event_types", typeID}, {"mappings", mappingID}, {"bindings", bindingID}, {"captures", captureID}} {
|
||||
// Catalogue writes still default to the system org; capture ownership
|
||||
// must come from the persisted source, never a caller-supplied org.
|
||||
for _, record := range []struct{ table, id string }{{"event_types", typeID}, {"mappings", mappingID}, {"bindings", bindingID}} {
|
||||
if _, err := db.Pool.Exec(ctx, "UPDATE "+record.table+" SET org_id=$1 WHERE id=$2", owner, record.id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ids[owner] = captureID
|
||||
if owner == "acme" {
|
||||
run, err := db.EnqueueBindingTask(org.WithOrg(ctx, owner), "", "", "Org dispatch", "", "deploy", "acme-bot", bindingID, 1, nil)
|
||||
if err != nil || run.Org != owner || run.Identity != "acme-bot" {
|
||||
t.Fatalf("binding task %#v err %v", run, err)
|
||||
}
|
||||
request := httptest.NewRequestWithContext(ctx, http.MethodPost, "/webhooks/capture/"+path, strings.NewReader("{}"))
|
||||
request.Header.Set("X-Org", string(owner))
|
||||
response := httptest.NewRecorder()
|
||||
mux.ServeHTTP(response, request)
|
||||
if response.Code != http.StatusAccepted {
|
||||
t.Fatalf("capture: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
var captureID, captureOrg, workspace string
|
||||
if err := db.Pool.QueryRow(ctx, "SELECT id, org_id, workspace_id FROM captures WHERE source=$1", path).Scan(&captureID, &captureOrg, &workspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if captureOrg != string(owner) || workspace != "incoming" {
|
||||
t.Fatalf("capture org %s workspace %s, want %s/incoming", captureOrg, workspace, owner)
|
||||
}
|
||||
ids[owner] = captureID
|
||||
bindings[owner] = bindingID
|
||||
}
|
||||
for _, owner := range []org.OrgID{org.DefaultOrgID, "acme"} {
|
||||
t.Run(string(owner), func(t *testing.T) {
|
||||
captures, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), []string{"webhook"}, 1)
|
||||
captures, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), paths, 1)
|
||||
if err != nil || len(captures) != 1 || captures[0].ID != ids[owner] {
|
||||
t.Fatalf("org %s got captures %#v err %v", owner, captures, err)
|
||||
}
|
||||
bindingID := bindings[owner]
|
||||
if err := client.RecordDispatch(t.Context(), bindingID, 1, ids[owner], 0, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
taskIdentity := ""
|
||||
if owner == "acme" {
|
||||
taskIdentity = "acme-bot"
|
||||
}
|
||||
run, err := client.EnqueueBindingTask(t.Context(), "", "", "Org dispatch", "", "deploy", taskIdentity, bindingID, 1, nil)
|
||||
if err != nil || run.Org != owner {
|
||||
t.Fatalf("binding task %#v err %v", run, err)
|
||||
}
|
||||
if err := client.SetDispatchTask(t.Context(), bindingID, ids[owner], run.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pending, err := eda.ListUndispatchedCaptures(org.WithOrg(t.Context(), owner), paths, 1)
|
||||
if err != nil || len(pending) != 0 {
|
||||
t.Fatalf("dispatched capture still pending: %#v err %v", pending, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("unknown source", func(t *testing.T) {
|
||||
id, err := client.InsertCapture(org.WithOrg(t.Context(), "acme"), storecontract.CapturedEvent{Source: "unknown", Body: "{}"}, time.Hour, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var owner, workspace string
|
||||
if err := db.Pool.QueryRow(t.Context(), "SELECT org_id,workspace_id FROM captures WHERE id=$1", id).Scan(&owner, &workspace); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if owner != string(org.DefaultOrgID) || workspace != string(org.DefaultWorkspaceID) {
|
||||
t.Fatalf("unknown source owner %s/%s", owner, workspace)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func captureRPC(t *testing.T, eda *postgres.EDA, db *pgstore.TaskDB) *staterpc.Client {
|
||||
t.Helper()
|
||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = listener.Close() })
|
||||
grants := &staterpc.TaskGrants{}
|
||||
srv := grpc.NewServer(grpc.ChainUnaryInterceptor(grants.UnaryInterceptor("test-token")), grpc.ChainStreamInterceptor(grants.StreamInterceptor("test-token")))
|
||||
staterpc.RegisterServer(srv, staterpc.Deps{Tasks: db, Captures: eda, Sources: eda, BindingDispatcher: eda, BindingTaskCreator: db})
|
||||
t.Cleanup(srv.Stop)
|
||||
go func() { _ = srv.Serve(listener) }()
|
||||
client, closeClient, err := staterpc.Dial(listener.Addr().String(), "archied", "test-token")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(closeClient)
|
||||
return client
|
||||
}
|
||||
@@ -91,17 +91,19 @@ func (s *EDA) InsertCapture(ctx context.Context, c storecontract.CapturedEvent,
|
||||
delivery = id
|
||||
}
|
||||
if err := s.q.InsertCapture(ctx, postgresdb.InsertCaptureParams{
|
||||
ID: id,
|
||||
Source: c.Source,
|
||||
RemoteAddr: c.RemoteAddr,
|
||||
ContentType: c.ContentType,
|
||||
Headers: c.Headers,
|
||||
Body: c.Body,
|
||||
Authenticated: c.Authenticated,
|
||||
Unsigned: c.Unsigned,
|
||||
ReceivedAt: received.UTC(),
|
||||
EventType: eventType,
|
||||
Delivery: delivery,
|
||||
ID: id,
|
||||
Source: c.Source,
|
||||
RemoteAddr: c.RemoteAddr,
|
||||
ContentType: c.ContentType,
|
||||
Headers: c.Headers,
|
||||
Body: c.Body,
|
||||
Authenticated: c.Authenticated,
|
||||
Unsigned: c.Unsigned,
|
||||
ReceivedAt: received.UTC(),
|
||||
EventType: eventType,
|
||||
Delivery: delivery,
|
||||
DefaultOrg: string(org.DefaultOrgID),
|
||||
DefaultWorkspace: string(org.DefaultWorkspaceID),
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("edastore: insert capture: %w", err)
|
||||
}
|
||||
|
||||
@@ -443,22 +443,26 @@ func (q *Queries) InsertBindingDispatch(ctx context.Context, arg InsertBindingDi
|
||||
|
||||
const insertCapture = `-- name: InsertCapture :exec
|
||||
|
||||
INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery, org_id, workspace_id)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
|
||||
COALESCE(s.org_id, $12::text), COALESCE(s.workspace_id, $13::text)
|
||||
FROM (SELECT 1) intake LEFT JOIN sources s ON s.path = $2
|
||||
`
|
||||
|
||||
type InsertCaptureParams struct {
|
||||
ID string
|
||||
Source string
|
||||
RemoteAddr string
|
||||
ContentType string
|
||||
Headers string
|
||||
Body string
|
||||
Authenticated bool
|
||||
ReceivedAt time.Time
|
||||
Unsigned bool
|
||||
EventType string
|
||||
Delivery string
|
||||
ID string
|
||||
Source string
|
||||
RemoteAddr string
|
||||
ContentType string
|
||||
Headers string
|
||||
Body string
|
||||
Authenticated bool
|
||||
ReceivedAt time.Time
|
||||
Unsigned bool
|
||||
EventType string
|
||||
Delivery string
|
||||
DefaultOrg string
|
||||
DefaultWorkspace string
|
||||
}
|
||||
|
||||
// EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls.
|
||||
@@ -475,6 +479,8 @@ func (q *Queries) InsertCapture(ctx context.Context, arg InsertCaptureParams) er
|
||||
arg.Unsigned,
|
||||
arg.EventType,
|
||||
arg.Delivery,
|
||||
arg.DefaultOrg,
|
||||
arg.DefaultWorkspace,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
-- EDA queries: captures, mappings, bindings, dispatch ledgers, tool_calls.
|
||||
|
||||
-- name: InsertCapture :exec
|
||||
INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11);
|
||||
INSERT INTO captures (id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, delivery, org_id, workspace_id)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
|
||||
COALESCE(s.org_id, @default_org::text), COALESCE(s.workspace_id, @default_workspace::text)
|
||||
FROM (SELECT 1) intake LEFT JOIN sources s ON s.path = $2;
|
||||
|
||||
-- name: ListCaptures :many
|
||||
SELECT id, source, remote_addr, content_type, headers, body, authenticated, received_at, unsigned, event_type, org_id, workspace_id, delivery
|
||||
|
||||
Reference in new issue
Block a user