fix(eda): the undispatched-capture batch lists only captures an enabled binding still owes
Issue: archie-core-kfjz1
This commit is contained in:
1 parent
df20b64f62
commit
cf5674f5da
10 files changed
+49
-37
No files matched your search
@@ -12189,9 +12189,11 @@ func (*DeletePlaybookDispatchesResponse) Descriptor() ([]byte, []int) {
|
||||
}
|
||||
|
||||
type StreamUndispatchedCapturesRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Sources []string `protobuf:"bytes,1,rep,name=sources,proto3" json:"sources,omitempty"`
|
||||
Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"`
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
// Bindings are the ids of the armed bindings, on enabled workflows, the
|
||||
// captures are listed for.
|
||||
Bindings []string `protobuf:"bytes,3,rep,name=bindings,proto3" json:"bindings,omitempty"`
|
||||
Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -12226,9 +12228,9 @@ func (*StreamUndispatchedCapturesRequest) Descriptor() ([]byte, []int) {
|
||||
return file_state_v1_state_proto_rawDescGZIP(), []int{223}
|
||||
}
|
||||
|
||||
func (x *StreamUndispatchedCapturesRequest) GetSources() []string {
|
||||
func (x *StreamUndispatchedCapturesRequest) GetBindings() []string {
|
||||
if x != nil {
|
||||
return x.Sources
|
||||
return x.Bindings
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -18421,10 +18423,10 @@ const file_state_v1_state_proto_rawDesc = "" +
|
||||
"\x1fDeletePlaybookDispatchesRequest\x12\x1f\n" +
|
||||
"\vplaybook_id\x18\x01 \x01(\tR\n" +
|
||||
"playbookId\"\"\n" +
|
||||
" DeletePlaybookDispatchesResponse\"S\n" +
|
||||
"!StreamUndispatchedCapturesRequest\x12\x18\n" +
|
||||
"\asources\x18\x01 \x03(\tR\asources\x12\x14\n" +
|
||||
"\x05limit\x18\x02 \x01(\x03R\x05limit\"W\n" +
|
||||
" DeletePlaybookDispatchesResponse\"d\n" +
|
||||
"!StreamUndispatchedCapturesRequest\x12\x1a\n" +
|
||||
"\bbindings\x18\x03 \x03(\tR\bbindings\x12\x14\n" +
|
||||
"\x05limit\x18\x02 \x01(\x03R\x05limitJ\x04\b\x01\x10\x02R\asources\"W\n" +
|
||||
"\"StreamUndispatchedCapturesResponse\x121\n" +
|
||||
"\acapture\x18\x01 \x01(\v2\x17.state.v1.CapturedEventR\acapture\"\xaf\x02\n" +
|
||||
"\x19EnqueueBindingTaskRequest\x12\x14\n" +
|
||||
|
||||
@@ -598,9 +598,9 @@ func (d *Daemon) dispatchBindings(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
|
||||
// Gather the sources of armed bindings whose workflow is enabled first, so
|
||||
// the oldest-first captures query is not held by captures that wait on a
|
||||
// disabled workflow.
|
||||
// Gather the armed bindings whose workflow is enabled first, so the
|
||||
// oldest-first captures query is not held by captures that only a binding
|
||||
// on a disabled workflow would take.
|
||||
bindings, err := d.Bindings.ListBindings(ctx)
|
||||
if err != nil {
|
||||
d.Log.Warn("list bindings failed", "error", err)
|
||||
@@ -611,17 +611,17 @@ func (d *Daemon) dispatchOrgBindings(ctx context.Context) {
|
||||
d.Log.Warn("binding dispatch: workflow enablement unavailable", "error", err)
|
||||
return
|
||||
}
|
||||
sources := make([]string, 0, len(bindings))
|
||||
enabled := make([]string, 0, len(bindings))
|
||||
for _, b := range bindings {
|
||||
if b.OrgID == org.OrgFromContext(ctx) && b.Status == binding.StatusArmed && b.Matcher.Source != "" && enablement.Enabled(b.OrgID, b.Workflow) {
|
||||
sources = append(sources, b.Matcher.Source)
|
||||
enabled = append(enabled, b.ID)
|
||||
}
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
if len(enabled) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, sources, bindingDispatchBatchLimit)
|
||||
captures, err := d.BindingDispatcher.ListUndispatchedCaptures(ctx, enabled, bindingDispatchBatchLimit)
|
||||
if err != nil {
|
||||
d.Log.Warn("list undispatched captures failed", "error", err)
|
||||
return
|
||||
|
||||
@@ -46,10 +46,12 @@ func (s *orgBindingStore) ArmedBindingsForSource(_ context.Context, source strin
|
||||
return slices.DeleteFunc(slices.Clone(s.bindings), func(b binding.Binding) bool { return b.Matcher.Source != source }), nil
|
||||
}
|
||||
|
||||
func (*orgBindingStore) ListUndispatchedCaptures(_ context.Context, sources []string, _ int) ([]storecontract.CapturedEvent, error) {
|
||||
func (s *orgBindingStore) ListUndispatchedCaptures(_ context.Context, bindings []string, _ int) ([]storecontract.CapturedEvent, error) {
|
||||
var captures []storecontract.CapturedEvent
|
||||
for _, source := range sources {
|
||||
captures = append(captures, storecontract.CapturedEvent{ID: source, Source: source, EventType: source, Body: "{}", Authenticated: true})
|
||||
for _, b := range s.bindings {
|
||||
if slices.Contains(bindings, b.ID) {
|
||||
captures = append(captures, storecontract.CapturedEvent{ID: b.Matcher.Source, Source: b.Matcher.Source, EventType: b.Matcher.Source, Body: "{}", Authenticated: true})
|
||||
}
|
||||
}
|
||||
return captures, nil
|
||||
}
|
||||
|
||||
@@ -331,7 +331,7 @@ type BindingDispatcher interface {
|
||||
// an evaluation that started no task.
|
||||
reason string,
|
||||
) error
|
||||
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
|
||||
ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]CapturedEvent, error)
|
||||
}
|
||||
|
||||
// BindingTaskCreator is the single-method consumer-facing surface for
|
||||
|
||||
@@ -166,16 +166,16 @@ func (s *EDA) ListCaptures(ctx context.Context, limit int) ([]storecontract.Capt
|
||||
}
|
||||
|
||||
// ListUndispatchedCaptures returns the oldest dispatchable, identified
|
||||
// captures for the given sources that some armed binding for their event type
|
||||
// has not dispatched yet. An unidentified capture is never returned, so it
|
||||
// captures that one of the given armed bindings, for their event type, has not
|
||||
// dispatched yet. An unidentified capture is never returned, so it
|
||||
// never dispatches.
|
||||
func (s *EDA) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) {
|
||||
if len(sources) == 0 || limit <= 0 {
|
||||
func (s *EDA) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) {
|
||||
if len(bindings) == 0 || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.q.ListUndispatchedCaptures(ctx, postgresdb.ListUndispatchedCapturesParams{
|
||||
OrgID: string(org.OrgFromContext(ctx)),
|
||||
Sources: sources,
|
||||
Bindings: bindings,
|
||||
EntryLimit: int32(limit),
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -1024,12 +1024,13 @@ func (q *Queries) ListSources(ctx context.Context, scopeOrg pgtype.Text) ([]Sour
|
||||
const listUndispatchedCaptures = `-- name: ListUndispatchedCaptures :many
|
||||
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery
|
||||
FROM captures c
|
||||
WHERE c.org_id = $1 AND c.source = ANY($2::text[])
|
||||
WHERE c.org_id = $1
|
||||
AND c.event_type <> ''
|
||||
AND (c.authenticated OR c.unsigned)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
|
||||
WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
|
||||
AND b.id = ANY($2::text[])
|
||||
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery)
|
||||
)
|
||||
ORDER BY c.received_at, c.id
|
||||
@@ -1038,15 +1039,16 @@ LIMIT $3
|
||||
|
||||
type ListUndispatchedCapturesParams struct {
|
||||
OrgID string
|
||||
Sources []string
|
||||
Bindings []string
|
||||
EntryLimit int32
|
||||
}
|
||||
|
||||
// Oldest first, so sustained intake cannot starve accepted work. Only
|
||||
// dispatchable captures (signed, or from an approved unsigned source) are
|
||||
// listed: no binding ever matches the rest, so they would hold the batch.
|
||||
// dispatchable captures (signed, or from an approved unsigned source) that one
|
||||
// of the named bindings still owes a dispatch are listed: no other capture
|
||||
// would ever be dispatched, so it would hold the batch.
|
||||
func (q *Queries) ListUndispatchedCaptures(ctx context.Context, arg ListUndispatchedCapturesParams) ([]Capture, error) {
|
||||
rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Sources, arg.EntryLimit)
|
||||
rows, err := q.db.Query(ctx, listUndispatchedCaptures, arg.OrgID, arg.Bindings, arg.EntryLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -15,16 +15,18 @@ LIMIT $1;
|
||||
|
||||
-- name: ListUndispatchedCaptures :many
|
||||
-- Oldest first, so sustained intake cannot starve accepted work. Only
|
||||
-- dispatchable captures (signed, or from an approved unsigned source) are
|
||||
-- listed: no binding ever matches the rest, so they would hold the batch.
|
||||
-- dispatchable captures (signed, or from an approved unsigned source) that one
|
||||
-- of the named bindings still owes a dispatch are listed: no other capture
|
||||
-- would ever be dispatched, so it would hold the batch.
|
||||
SELECT c.id, c.source, c.remote_addr, c.content_type, c.headers, c.body, c.authenticated, c.received_at, c.unsigned, c.event_type, c.org_id, c.workspace_id, c.delivery
|
||||
FROM captures c
|
||||
WHERE c.org_id = @org_id AND c.source = ANY(@sources::text[])
|
||||
WHERE c.org_id = @org_id
|
||||
AND c.event_type <> ''
|
||||
AND (c.authenticated OR c.unsigned)
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM bindings b JOIN mappings m ON m.id = b.mapping
|
||||
WHERE b.org_id = c.org_id AND b.source = c.source AND b.status = 'armed' AND m.event_type = c.event_type
|
||||
AND b.id = ANY(@bindings::text[])
|
||||
AND NOT EXISTS (SELECT 1 FROM binding_dispatches d WHERE d.binding = b.id AND d.delivery = c.delivery)
|
||||
)
|
||||
ORDER BY c.received_at, c.id
|
||||
|
||||
@@ -755,8 +755,8 @@ func (c *Client) DeletePlaybookDispatches(ctx context.Context, playbookID string
|
||||
}
|
||||
|
||||
// ListUndispatchedCaptures streams for the same reason as ListCaptures.
|
||||
func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]storecontract.CapturedEvent, error) {
|
||||
stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Sources: sources, Limit: int64(limit)})
|
||||
func (c *Client) ListUndispatchedCaptures(ctx context.Context, bindings []string, limit int) ([]storecontract.CapturedEvent, error) {
|
||||
stream, err := c.client.StreamUndispatchedCaptures(ctx, &pb.StreamUndispatchedCapturesRequest{Bindings: bindings, Limit: int64(limit)})
|
||||
if err != nil {
|
||||
return nil, unmapError(err)
|
||||
}
|
||||
|
||||
@@ -1186,7 +1186,7 @@ func (s *server) StreamUndispatchedCaptures(r *pb.StreamUndispatchedCapturesRequ
|
||||
if s.deps.BindingDispatcher == nil {
|
||||
return errBindingDispatchUnavailable
|
||||
}
|
||||
captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Sources, int(r.Limit))
|
||||
captures, err := s.deps.BindingDispatcher.ListUndispatchedCaptures(stream.Context(), r.Bindings, int(r.Limit))
|
||||
if err != nil {
|
||||
return s.logErr("StreamUndispatchedCaptures", err)
|
||||
}
|
||||
|
||||
@@ -1313,7 +1313,11 @@ message DeletePlaybookDispatchesRequest { string playbook_id = 1; }
|
||||
message DeletePlaybookDispatchesResponse {}
|
||||
|
||||
message StreamUndispatchedCapturesRequest {
|
||||
repeated string sources = 1;
|
||||
reserved 1;
|
||||
reserved "sources";
|
||||
// Bindings are the ids of the armed bindings, on enabled workflows, the
|
||||
// captures are listed for.
|
||||
repeated string bindings = 3;
|
||||
int64 limit = 2;
|
||||
}
|
||||
// Streamed one capture per message; see StreamUndispatchedCaptures above.
|
||||
|
||||
Reference in new issue
Block a user