From cf5674f5da365db073ccaef682e4f85e7e7878c2 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 13 Oct 2026 16:21:21 +1100 Subject: [PATCH] fix(eda): the undispatched-capture batch lists only captures an enabled binding still owes Issue: archie-core-kfjz1 --- internal/contracts/state/v1/state.pb.go | 20 ++++++++++--------- internal/daemon/daemon.go | 14 ++++++------- internal/daemon/workflow_org_test.go | 8 +++++--- .../domain/storecontract/storecontract.go | 2 +- internal/infrastructure/postgres/edastore.go | 10 +++++----- .../postgres/postgresdb/edastore.sql.go | 12 ++++++----- .../postgres/queries/edastore.sql | 8 +++++--- internal/infrastructure/staterpc/client.go | 4 ++-- internal/infrastructure/staterpc/server.go | 2 +- proto/state/v1/state.proto | 6 +++++- 10 files changed, 49 insertions(+), 37 deletions(-) diff --git a/internal/contracts/state/v1/state.pb.go b/internal/contracts/state/v1/state.pb.go index d33ac75d5..678ff44b5 100644 --- a/internal/contracts/state/v1/state.pb.go +++ b/internal/contracts/state/v1/state.pb.go @@ -12189,9 +12189,11 @@ func (*DeletePlaybookDispatchesResponse) Descriptor() ([]byte, []int) { } type StreamUndispatchedCapturesRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Sources []string `protobuf:"bytes,1,rep,name=sources,proto3" json:"sources,omitempty"` - Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // Bindings are the ids of the armed bindings, on enabled workflows, the + // captures are listed for. + Bindings []string `protobuf:"bytes,3,rep,name=bindings,proto3" json:"bindings,omitempty"` + Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -12226,9 +12228,9 @@ func (*StreamUndispatchedCapturesRequest) Descriptor() ([]byte, []int) { return file_state_v1_state_proto_rawDescGZIP(), []int{223} } -func (x *StreamUndispatchedCapturesRequest) GetSources() []string { +func (x *StreamUndispatchedCapturesRequest) GetBindings() []string { if x != nil { - return x.Sources + return x.Bindings } return nil } @@ -18421,10 +18423,10 @@ const file_state_v1_state_proto_rawDesc = "" + "\x1fDeletePlaybookDispatchesRequest\x12\x1f\n" + "\vplaybook_id\x18\x01 \x01(\tR\n" + "playbookId\"\"\n" + - " DeletePlaybookDispatchesResponse\"S\n" + - "!StreamUndispatchedCapturesRequest\x12\x18\n" + - "\asources\x18\x01 \x03(\tR\asources\x12\x14\n" + - "\x05limit\x18\x02 \x01(\x03R\x05limit\"W\n" + + " DeletePlaybookDispatchesResponse\"d\n" + + "!StreamUndispatchedCapturesRequest\x12\x1a\n" + + "\bbindings\x18\x03 \x03(\tR\bbindings\x12\x14\n" + + "\x05limit\x18\x02 \x01(\x03R\x05limitJ\x04\b\x01\x10\x02R\asources\"W\n" + "\"StreamUndispatchedCapturesResponse\x121\n" + "\acapture\x18\x01 \x01(\v2\x17.state.v1.CapturedEventR\acapture\"\xaf\x02\n" + "\x19EnqueueBindingTaskRequest\x12\x14\n" + diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 11dae59e4..a4183190b 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -598,9 +598,9 @@ func (d *Daemon) dispatchBindings(ctx context.Context) { } func (d *Daemon) dispatchOrgBindings(ctx context.Context) { - // Gather the sources of armed bindings whose workflow is enabled first, so - // the oldest-first captures query is not held by captures that wait on a - // disabled workflow. + // Gather the armed bindings whose workflow is enabled first, so the + // oldest-first captures query is not held by captures that only a binding + // on a disabled workflow would take. bindings, err := d.Bindings.ListBindings(ctx) if err != nil { d.Log.Warn("list bindings failed", "error", err) @@ -611,17 +611,17 @@ func (d *Daemon) dispatchOrgBindings(ctx context.Context) { d.Log.Warn("binding dispatch: workflow enablement unavailable", "error", err) return } - sources := make([]string, 0, len(bindings)) + enabled := make([]string, 0, len(bindings)) for _, b := range bindings { if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) { - sources = append(sources, b.Matcher.Source) + enabled = append(enabled, b.ID) } } - if len(sources) == 0 { + if len(enabled) == 0 { return } - captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, sources, bindingDispatchBatchLimit) + captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit) if err != nil { d.Log.Warn("list undispatched captures failed", "error", err) return diff --git a/internal/daemon/workflow_org_test.go b/internal/daemon/workflow_org_test.go index 343415bbd..8b99aaa97 100644 --- a/internal/daemon/workflow_org_test.go +++ b/internal/daemon/workflow_org_test.go @@ -46,10 +46,12 @@ func (s *orgBindingStore) ArmedBindingsForSource(_ context.Context, source strin return slices.DeleteFunc(slices.Clone(s.bindings), func(b binding.Binding) bool { return b.Matcher.Source != source }), nil } -func (*orgBindingStore) ListUndispatchedCaptures(_ context.Context, sources []string, _ int) ([]storecontract.CapturedEvent, error) { +func (s *orgBindingStore) ListUndispatchedCaptures(_ context.Context, bindings []string, _ int) ([]storecontract.CapturedEvent, error) { var captures []storecontract.CapturedEvent - for _, source := range sources { - captures = append(captures, storecontract.CapturedEvent{ID: source, Source: source, EventType: source, Body: "{}", Authenticated: true}) + for _, b := range s.bindings { + if slices.Contains(bindings, b.ID) { + captures = append(captures, storecontract.CapturedEvent{ID: b.Matcher.Source, Source: b.Matcher.Source, EventType: b.Matcher.Source, Body: "{}", Authenticated: true}) + } } return captures, nil } diff --git a/internal/domain/storecontract/storecontract.go b/internal/domain/storecontract/storecontract.go index 86cd278a3..240651931 100644 --- a/internal/domain/storecontract/storecontract.go +++ b/internal/domain/storecontract/storecontract.go @@ -331,7 +331,7 @@ type BindingDispatcher interface { // an evaluation that started no task. reason string, ) error - ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error) + ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]CapturedEvent, error) } // BindingTaskCreator is the single-method consumer-facing surface for diff --git a/internal/infrastructure/postgres/edastore.go b/internal/infrastructure/postgres/edastore.go index 92b4f43db..6fe6bdcbd 100644 --- a/internal/infrastructure/postgres/edastore.go +++ b/internal/infrastructure/postgres/edastore.go @@ -166,16 +166,16 @@ func (s *EDA) ListCaptures(ctx context.Context, limit int) ([]storecontract.Capt } // ListUndispatchedCaptures returns the oldest dispatchable, identified -// captures for the given sources that some armed binding for their event type -// has not dispatched yet. An unidentified capture is never returned, so it +// captures that one of the given armed bindings, for their event type, has not +// dispatched yet. An unidentified capture is never returned, so it // never dispatches. -func (s *EDA) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) { - if len(sources) == 0 || limit <= 0 { +func (s *EDA) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) { + if len(bindings) == 0 || limit <= 0 { return nil, nil } rows, err := s.q.ListUndispatchedCaptures(ctx, postgresdb.ListUndispatchedCapturesParams{ OrgID: string(org.OrgFromContext(ctx)), - Sources: sources, + Bindings: bindings, EntryLimit: int32(limit), }) if err != nil { diff --git a/internal/infrastructure/postgres/postgresdb/edastore.sql.go b/internal/infrastructure/postgres/postgresdb/edastore.sql.go index 3e538eef0..cfbd72803 100644 --- a/internal/infrastructure/postgres/postgresdb/edastore.sql.go +++ b/internal/infrastructure/postgres/postgresdb/edastore.sql.go @@ -1024,12 +1024,13 @@ func (q *Queries) ListSources(ctx context.Context, scopeOrg pgtype.Text) ([]Sour const listUndispatchedCaptures = `-- name: ListUndispatchedCaptures :many SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery FROM captures c -WHERE c.org_id = $1 AND c.source = ANY($2::text[]) +WHERE c.org_id = $1 AND c.event_type <> '' AND (c.authenticated OR c.unsigned) AND EXISTS ( SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type + AND b.id = ANY($2::text[]) AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery) ) ORDER BY c.received_at, c.id @@ -1038,15 +1039,16 @@ LIMIT $3 type ListUndispatchedCapturesParams struct { OrgID string - Sources []string + Bindings []string EntryLimit int32 } // Oldest first, so sustained intake cannot starve accepted work. Only -// dispatchable captures (signed, or from an approved unsigned source) are -// listed: no binding ever matches the rest, so they would hold the batch. +// dispatchable captures (signed, or from an approved unsigned source) that one +// of the named bindings still owes a dispatch are listed: no other capture +// would ever be dispatched, so it would hold the batch. func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispatchedCapturesParams) ([]Capture, error) { - rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Sources, arg.EntryLimit) + rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Bindings, arg.EntryLimit) if err != nil { return nil, err } diff --git a/internal/infrastructure/postgres/queries/edastore.sql b/internal/infrastructure/postgres/queries/edastore.sql index 50b29be2d..dc36c263d 100644 --- a/internal/infrastructure/postgres/queries/edastore.sql +++ b/internal/infrastructure/postgres/queries/edastore.sql @@ -15,16 +15,18 @@ LIMIT $1; -- name: ListUndispatchedCaptures :many -- Oldest first, so sustained intake cannot starve accepted work. Only --- dispatchable captures (signed, or from an approved unsigned source) are --- listed: no binding ever matches the rest, so they would hold the batch. +-- dispatchable captures (signed, or from an approved unsigned source) that one +-- of the named bindings still owes a dispatch are listed: no other capture +-- would ever be dispatched, so it would hold the batch. SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery FROM captures c -WHERE c.org_id = @org_id AND c.source = ANY(@sources::text[]) +WHERE c.org_id = @org_id AND c.event_type <> '' AND (c.authenticated OR c.unsigned) AND EXISTS ( SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type + AND b.id = ANY(@bindings::text[]) AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery) ) ORDER BY c.received_at, c.id diff --git a/internal/infrastructure/staterpc/client.go b/internal/infrastructure/staterpc/client.go index 3438f1e98..6380e9f06 100644 --- a/internal/infrastructure/staterpc/client.go +++ b/internal/infrastructure/staterpc/client.go @@ -755,8 +755,8 @@ func (c *Client) DeletePlaybookDispatches(ctx context.Context, playbookID string } // ListUndispatchedCaptures streams for the same reason as ListCaptures. -func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) { - stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Sources: sources, Limit: int64(limit)}) +func (c *Client) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) { + stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Bindings: bindings, Limit: int64(limit)}) if err != nil { return nil, unmapError(err) } diff --git a/internal/infrastructure/staterpc/server.go b/internal/infrastructure/staterpc/server.go index 6c494726b..33de11a0a 100644 --- a/internal/infrastructure/staterpc/server.go +++ b/internal/infrastructure/staterpc/server.go @@ -1186,7 +1186,7 @@ func (s *server) StreamUndispatchedCaptures(r *pb.StreamUndispatchedCapturesRequ if s.deps.BindingDispatcher == nil { return errBindingDispatchUnavailable } - captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Sources, int(r.Limit)) + captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Bindings, int(r.Limit)) if err != nil { return s.logErr("StreamUndispatchedCaptures", err) } diff --git a/proto/state/v1/state.proto b/proto/state/v1/state.proto index f8730dd5a..e6358ae47 100644 --- a/proto/state/v1/state.proto +++ b/proto/state/v1/state.proto @@ -1313,7 +1313,11 @@ message DeletePlaybookDispatchesRequest { string playbook_id = 1; } message DeletePlaybookDispatchesResponse {} message StreamUndispatchedCapturesRequest { - repeated string sources = 1; + reserved 1; + reserved "sources"; + // Bindings are the ids of the armed bindings, on enabled workflows, the + // captures are listed for. + repeated string bindings = 3; int64 limit = 2; } // Streamed one capture per message; see StreamUndispatchedCaptures above.