fix(eda): the undispatched-capture batch lists only captures an enabled binding still owes

Issue: archie-core-kfjz1
This commit is contained in:
sam committed 2026-10-13 16:21:21 +11:00
1 parent 796c7a711b
commit 7435b6efe5
10 files changed
+49 -37

No files matched your search

+11 -9
View File
@@ -12189,9 +12189,11 @@ func (*DeletePlaybookDispatchesResponse) Descriptor() ([]byte, []int) {
}
type StreamUndispatchedCapturesRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
Sources []string `protobuf:"bytes,1,rep,name=sources,proto3" json:"sources,omitempty"`
Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"`
state protoimpl.MessageState `protogen:"open.v1"`
// Bindings are the ids of the armed bindings, on enabled workflows, the
// captures are listed for.
Bindings []string `protobuf:"bytes,3,rep,name=bindings,proto3" json:"bindings,omitempty"`
Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -12226,9 +12228,9 @@ func (*StreamUndispatchedCapturesRequest) Descriptor() ([]byte, []int) {
return file_state_v1_state_proto_rawDescGZIP(), []int{223}
}
func (x *StreamUndispatchedCapturesRequest) GetSources() []string {
func (x *StreamUndispatchedCapturesRequest) GetBindings() []string {
if x != nil {
return x.Sources
return x.Bindings
}
return nil
}
@@ -18421,10 +18423,10 @@ const file_state_v1_state_proto_rawDesc = "" +
"\x1fDeletePlaybookDispatchesRequest\x12\x1f\n" +
"\vplaybook_id\x18\x01 \x01(\tR\n" +
"playbookId\"\"\n" +
" DeletePlaybookDispatchesResponse\"S\n" +
"!StreamUndispatchedCapturesRequest\x12\x18\n" +
"\asources\x18\x01 \x03(\tR\asources\x12\x14\n" +
"\x05limit\x18\x02 \x01(\x03R\x05limit\"W\n" +
" DeletePlaybookDispatchesResponse\"d\n" +
"!StreamUndispatchedCapturesRequest\x12\x1a\n" +
"\bbindings\x18\x03 \x03(\tR\bbindings\x12\x14\n" +
"\x05limit\x18\x02 \x01(\x03R\x05limitJ\x04\b\x01\x10\x02R\asources\"W\n" +
"\"StreamUndispatchedCapturesResponse\x121\n" +
"\acapture\x18\x01 \x01(\v2\x17.state.v1.CapturedEventR\acapture\"\xaf\x02\n" +
"\x19EnqueueBindingTaskRequest\x12\x14\n" +
+7 -7
View File
@@ -598,9 +598,9 @@ func (d *Daemon) dispatchBindings(ctx context.Context) {
}
func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
// Gather the sources of armed bindings whose workflow is enabled first, so
// the oldest-first captures query is not held by captures that wait on a
// disabled workflow.
// Gather the armed bindings whose workflow is enabled first, so the
// oldest-first captures query is not held by captures that only a binding
// on a disabled workflow would take.
bindings, err := d.Bindings.ListBindings(ctx)
if err != nil {
d.Log.Warn("list bindings failed", "error", err)
@@ -611,17 +611,17 @@ func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
d.Log.Warn("binding dispatch: workflow enablement unavailable", "error", err)
return
}
sources := make([]string, 0, len(bindings))
enabled := make([]string, 0, len(bindings))
for _, b := range bindings {
if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) {
sources = append(sources, b.Matcher.Source)
enabled = append(enabled, b.ID)
}
}
if len(sources) == 0 {
if len(enabled) == 0 {
return
}
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, sources, bindingDispatchBatchLimit)
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit)
if err != nil {
d.Log.Warn("list undispatched captures failed", "error", err)
return
+5 -3
View File
@@ -46,10 +46,12 @@ func (s *orgBindingStore) ArmedBindingsForSource(_ context.Context, source strin
return slices.DeleteFunc(slices.Clone(s.bindings), func(b binding.Binding) bool { return b.Matcher.Source != source }), nil
}
func (*orgBindingStore) ListUndispatchedCaptures(_ context.Context, sources []string, _ int) ([]storecontract.CapturedEvent, error) {
func (s *orgBindingStore) ListUndispatchedCaptures(_ context.Context, bindings []string, _ int) ([]storecontract.CapturedEvent, error) {
var captures []storecontract.CapturedEvent
for _, source := range sources {
captures = append(captures, storecontract.CapturedEvent{ID: source, Source: source, EventType: source, Body: "{}", Authenticated: true})
for _, b := range s.bindings {
if slices.Contains(bindings, b.ID) {
captures = append(captures, storecontract.CapturedEvent{ID: b.Matcher.Source, Source: b.Matcher.Source, EventType: b.Matcher.Source, Body: "{}", Authenticated: true})
}
}
return captures, nil
}
@@ -331,7 +331,7 @@ type BindingDispatcher interface {
// an evaluation that started no task.
reason string,
) error
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]CapturedEvent, error)
}
// BindingTaskCreator is the single-method consumer-facing surface for
+5 -5
View File
@@ -166,16 +166,16 @@ func (s *EDA) ListCaptures(ctx context.Context, limit int) ([]storecontract.Capt
}
// ListUndispatchedCaptures returns the oldest dispatchable, identified
// captures for the given sources that some armed binding for their event type
// has not dispatched yet. An unidentified capture is never returned, so it
// captures that one of the given armed bindings, for their event type, has not
// dispatched yet. An unidentified capture is never returned, so it
// never dispatches.
func (s *EDA) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) {
if len(sources) == 0 || limit <= 0 {
func (s *EDA) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) {
if len(bindings) == 0 || limit <= 0 {
return nil, nil
}
rows, err := s.q.ListUndispatchedCaptures(ctx, postgresdb.ListUndispatchedCapturesParams{
OrgID: string(org.OrgFromContext(ctx)),
Sources: sources,
Bindings: bindings,
EntryLimit: int32(limit),
})
if err != nil {
@@ -1024,12 +1024,13 @@ func (q *Queries) ListSources(ctx context.Context, scopeOrg pgtype.Text) ([]Sour
const listUndispatchedCaptures = `-- name: ListUndispatchedCaptures :many
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery
FROM captures c
WHERE c.org_id = $1 AND c.source = ANY($2::text[])
WHERE c.org_id = $1
AND c.event_type <> ''
AND (c.authenticated OR c.unsigned)
AND EXISTS (
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
AND b.id = ANY($2::text[])
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery)
)
ORDER BY c.received_at, c.id
@@ -1038,15 +1039,16 @@ LIMIT $3
type ListUndispatchedCapturesParams struct {
OrgID string
Sources []string
Bindings []string
EntryLimit int32
}
// Oldest first, so sustained intake cannot starve accepted work. Only
// dispatchable captures (signed, or from an approved unsigned source) are
// listed: no binding ever matches the rest, so they would hold the batch.
// dispatchable captures (signed, or from an approved unsigned source) that one
// of the named bindings still owes a dispatch are listed: no other capture
// would ever be dispatched, so it would hold the batch.
func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispatchedCapturesParams) ([]Capture, error) {
rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Sources, arg.EntryLimit)
rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Bindings, arg.EntryLimit)
if err != nil {
return nil, err
}
@@ -15,16 +15,18 @@ LIMIT $1;
-- name: ListUndispatchedCaptures :many
-- Oldest first, so sustained intake cannot starve accepted work. Only
-- dispatchable captures (signed, or from an approved unsigned source) are
-- listed: no binding ever matches the rest, so they would hold the batch.
-- dispatchable captures (signed, or from an approved unsigned source) that one
-- of the named bindings still owes a dispatch are listed: no other capture
-- would ever be dispatched, so it would hold the batch.
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery
FROM captures c
WHERE c.org_id = @org_id AND c.source = ANY(@sources::text[])
WHERE c.org_id = @org_id
AND c.event_type <> ''
AND (c.authenticated OR c.unsigned)
AND EXISTS (
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
AND b.id = ANY(@bindings::text[])
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery)
)
ORDER BY c.received_at, c.id
+2 -2
View File
@@ -755,8 +755,8 @@ func (c *Client) DeletePlaybookDispatches(ctx context.Context, playbookID string
}
// ListUndispatchedCaptures streams for the same reason as ListCaptures.
func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) {
stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Sources: sources, Limit: int64(limit)})
func (c *Client) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) {
stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Bindings: bindings, Limit: int64(limit)})
if err != nil {
return nil, unmapError(err)
}
+1 -1
View File
@@ -1186,7 +1186,7 @@ func (s *server) StreamUndispatchedCaptures(r *pb.StreamUndispatchedCapturesRequ
if s.deps.BindingDispatcher == nil {
return errBindingDispatchUnavailable
}
captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Sources, int(r.Limit))
captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Bindings, int(r.Limit))
if err != nil {
return s.logErr("StreamUndispatchedCaptures", err)
}
+5 -1
View File
@@ -1313,7 +1313,11 @@ message DeletePlaybookDispatchesRequest { string playbook_id = 1; }
message DeletePlaybookDispatchesResponse {}
message StreamUndispatchedCapturesRequest {
repeated string sources = 1;
reserved 1;
reserved "sources";
// Bindings are the ids of the armed bindings, on enabled workflows, the
// captures are listed for.
repeated string bindings = 3;
int64 limit = 2;
}
// Streamed one capture per message; see StreamUndispatchedCaptures above.