fix(daemon): claim binding dispatch and enqueue its task atomically
EnqueueBindingTask now takes the capture and claims the ledger row, inserts the task, stamps its provenance and links the ledger row in one State Store transaction. A failure anywhere leaves the delivery undispatched for the next cycle; a replay or concurrent claim is refused by the ledger's unique index. SetDispatchTask is removed.
This commit is contained in:
11 files changed
+833
-837
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -130,7 +130,6 @@ const (
|
|||||||
StateStoreService_DeleteSource_FullMethodName = "/state.v1.StateStoreService/DeleteSource"
|
StateStoreService_DeleteSource_FullMethodName = "/state.v1.StateStoreService/DeleteSource"
|
||||||
StateStoreService_ArmedBindingsForSource_FullMethodName = "/state.v1.StateStoreService/ArmedBindingsForSource"
|
StateStoreService_ArmedBindingsForSource_FullMethodName = "/state.v1.StateStoreService/ArmedBindingsForSource"
|
||||||
StateStoreService_RecordDispatch_FullMethodName = "/state.v1.StateStoreService/RecordDispatch"
|
StateStoreService_RecordDispatch_FullMethodName = "/state.v1.StateStoreService/RecordDispatch"
|
||||||
StateStoreService_SetDispatchTask_FullMethodName = "/state.v1.StateStoreService/SetDispatchTask"
|
|
||||||
StateStoreService_RecordPlaybookDispatch_FullMethodName = "/state.v1.StateStoreService/RecordPlaybookDispatch"
|
StateStoreService_RecordPlaybookDispatch_FullMethodName = "/state.v1.StateStoreService/RecordPlaybookDispatch"
|
||||||
StateStoreService_DeletePlaybookDispatches_FullMethodName = "/state.v1.StateStoreService/DeletePlaybookDispatches"
|
StateStoreService_DeletePlaybookDispatches_FullMethodName = "/state.v1.StateStoreService/DeletePlaybookDispatches"
|
||||||
StateStoreService_StreamUndispatchedCaptures_FullMethodName = "/state.v1.StateStoreService/StreamUndispatchedCaptures"
|
StateStoreService_StreamUndispatchedCaptures_FullMethodName = "/state.v1.StateStoreService/StreamUndispatchedCaptures"
|
||||||
@@ -369,7 +368,6 @@ type StateStoreServiceClient interface {
|
|||||||
// Dispatch
|
// Dispatch
|
||||||
ArmedBindingsForSource(ctx context.Context, in *ArmedBindingsForSourceRequest, opts ...grpc.CallOption) (*ArmedBindingsForSourceResponse, error)
|
ArmedBindingsForSource(ctx context.Context, in *ArmedBindingsForSourceRequest, opts ...grpc.CallOption) (*ArmedBindingsForSourceResponse, error)
|
||||||
RecordDispatch(ctx context.Context, in *RecordDispatchRequest, opts ...grpc.CallOption) (*RecordDispatchResponse, error)
|
RecordDispatch(ctx context.Context, in *RecordDispatchRequest, opts ...grpc.CallOption) (*RecordDispatchResponse, error)
|
||||||
SetDispatchTask(ctx context.Context, in *SetDispatchTaskRequest, opts ...grpc.CallOption) (*SetDispatchTaskResponse, error)
|
|
||||||
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
||||||
// at-most-once record per (playbook, version, event, action) for the
|
// at-most-once record per (playbook, version, event, action) for the
|
||||||
// side-effecting positions. Administrative, like RecordDispatch: a
|
// side-effecting positions. Administrative, like RecordDispatch: a
|
||||||
@@ -377,7 +375,9 @@ type StateStoreServiceClient interface {
|
|||||||
RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error)
|
RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error)
|
||||||
DeletePlaybookDispatches(ctx context.Context, in *DeletePlaybookDispatchesRequest, opts ...grpc.CallOption) (*DeletePlaybookDispatchesResponse, error)
|
DeletePlaybookDispatches(ctx context.Context, in *DeletePlaybookDispatchesRequest, opts ...grpc.CallOption) (*DeletePlaybookDispatchesResponse, error)
|
||||||
StreamUndispatchedCaptures(ctx context.Context, in *StreamUndispatchedCapturesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamUndispatchedCapturesResponse], error)
|
StreamUndispatchedCaptures(ctx context.Context, in *StreamUndispatchedCapturesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamUndispatchedCapturesResponse], error)
|
||||||
// BindingTaskCreator
|
// BindingTaskCreator: claims the binding's dispatch of a capture and
|
||||||
|
// enqueues its task in one transaction, so a failure leaves neither and a
|
||||||
|
// replayed delivery is refused with AlreadyExists.
|
||||||
EnqueueBindingTask(ctx context.Context, in *EnqueueBindingTaskRequest, opts ...grpc.CallOption) (*EnqueueBindingTaskResponse, error)
|
EnqueueBindingTask(ctx context.Context, in *EnqueueBindingTaskRequest, opts ...grpc.CallOption) (*EnqueueBindingTaskResponse, error)
|
||||||
// WorkflowCaller: a workflow.call step starts its callee and, with
|
// WorkflowCaller: a workflow.call step starts its callee and, with
|
||||||
// wait:true, reads it back while waiting. Both are the one sanctioned
|
// wait:true, reads it back while waiting. Both are the one sanctioned
|
||||||
@@ -1551,16 +1551,6 @@ func (c *stateStoreServiceClient) RecordDispatch(ctx context.Context, in *Record
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *stateStoreServiceClient) SetDispatchTask(ctx context.Context, in *SetDispatchTaskRequest, opts ...grpc.CallOption) (*SetDispatchTaskResponse, error) {
|
|
||||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
|
||||||
out := new(SetDispatchTaskResponse)
|
|
||||||
err := c.cc.Invoke(ctx, StateStoreService_SetDispatchTask_FullMethodName, in, out, cOpts...)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *stateStoreServiceClient) RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error) {
|
func (c *stateStoreServiceClient) RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error) {
|
||||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||||
out := new(RecordPlaybookDispatchResponse)
|
out := new(RecordPlaybookDispatchResponse)
|
||||||
@@ -2004,7 +1994,6 @@ type StateStoreServiceServer interface {
|
|||||||
// Dispatch
|
// Dispatch
|
||||||
ArmedBindingsForSource(context.Context, *ArmedBindingsForSourceRequest) (*ArmedBindingsForSourceResponse, error)
|
ArmedBindingsForSource(context.Context, *ArmedBindingsForSourceRequest) (*ArmedBindingsForSourceResponse, error)
|
||||||
RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error)
|
RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error)
|
||||||
SetDispatchTask(context.Context, *SetDispatchTaskRequest) (*SetDispatchTaskResponse, error)
|
|
||||||
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
||||||
// at-most-once record per (playbook, version, event, action) for the
|
// at-most-once record per (playbook, version, event, action) for the
|
||||||
// side-effecting positions. Administrative, like RecordDispatch: a
|
// side-effecting positions. Administrative, like RecordDispatch: a
|
||||||
@@ -2012,7 +2001,9 @@ type StateStoreServiceServer interface {
|
|||||||
RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error)
|
RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error)
|
||||||
DeletePlaybookDispatches(context.Context, *DeletePlaybookDispatchesRequest) (*DeletePlaybookDispatchesResponse, error)
|
DeletePlaybookDispatches(context.Context, *DeletePlaybookDispatchesRequest) (*DeletePlaybookDispatchesResponse, error)
|
||||||
StreamUndispatchedCaptures(*StreamUndispatchedCapturesRequest, grpc.ServerStreamingServer[StreamUndispatchedCapturesResponse]) error
|
StreamUndispatchedCaptures(*StreamUndispatchedCapturesRequest, grpc.ServerStreamingServer[StreamUndispatchedCapturesResponse]) error
|
||||||
// BindingTaskCreator
|
// BindingTaskCreator: claims the binding's dispatch of a capture and
|
||||||
|
// enqueues its task in one transaction, so a failure leaves neither and a
|
||||||
|
// replayed delivery is refused with AlreadyExists.
|
||||||
EnqueueBindingTask(context.Context, *EnqueueBindingTaskRequest) (*EnqueueBindingTaskResponse, error)
|
EnqueueBindingTask(context.Context, *EnqueueBindingTaskRequest) (*EnqueueBindingTaskResponse, error)
|
||||||
// WorkflowCaller: a workflow.call step starts its callee and, with
|
// WorkflowCaller: a workflow.call step starts its callee and, with
|
||||||
// wait:true, reads it back while waiting. Both are the one sanctioned
|
// wait:true, reads it back while waiting. Both are the one sanctioned
|
||||||
@@ -2391,9 +2382,6 @@ func (UnimplementedStateStoreServiceServer) ArmedBindingsForSource(context.Conte
|
|||||||
func (UnimplementedStateStoreServiceServer) RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error) {
|
func (UnimplementedStateStoreServiceServer) RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "method RecordDispatch not implemented")
|
return nil, status.Error(codes.Unimplemented, "method RecordDispatch not implemented")
|
||||||
}
|
}
|
||||||
func (UnimplementedStateStoreServiceServer) SetDispatchTask(context.Context, *SetDispatchTaskRequest) (*SetDispatchTaskResponse, error) {
|
|
||||||
return nil, status.Error(codes.Unimplemented, "method SetDispatchTask not implemented")
|
|
||||||
}
|
|
||||||
func (UnimplementedStateStoreServiceServer) RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error) {
|
func (UnimplementedStateStoreServiceServer) RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error) {
|
||||||
return nil, status.Error(codes.Unimplemented, "method RecordPlaybookDispatch not implemented")
|
return nil, status.Error(codes.Unimplemented, "method RecordPlaybookDispatch not implemented")
|
||||||
}
|
}
|
||||||
@@ -4465,24 +4453,6 @@ func _StateStoreService_RecordDispatch_Handler(srv interface{}, ctx context.Cont
|
|||||||
return interceptor(ctx, in, info, handler)
|
return interceptor(ctx, in, info, handler)
|
||||||
}
|
}
|
||||||
|
|
||||||
func _StateStoreService_SetDispatchTask_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
|
||||||
in := new(SetDispatchTaskRequest)
|
|
||||||
if err := dec(in); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if interceptor == nil {
|
|
||||||
return srv.(StateStoreServiceServer).SetDispatchTask(ctx, in)
|
|
||||||
}
|
|
||||||
info := &grpc.UnaryServerInfo{
|
|
||||||
Server: srv,
|
|
||||||
FullMethod: StateStoreService_SetDispatchTask_FullMethodName,
|
|
||||||
}
|
|
||||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
|
||||||
return srv.(StateStoreServiceServer).SetDispatchTask(ctx, req.(*SetDispatchTaskRequest))
|
|
||||||
}
|
|
||||||
return interceptor(ctx, in, info, handler)
|
|
||||||
}
|
|
||||||
|
|
||||||
func _StateStoreService_RecordPlaybookDispatch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
func _StateStoreService_RecordPlaybookDispatch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||||
in := new(RecordPlaybookDispatchRequest)
|
in := new(RecordPlaybookDispatchRequest)
|
||||||
if err := dec(in); err != nil {
|
if err := dec(in); err != nil {
|
||||||
@@ -5315,10 +5285,6 @@ var StateStoreService_ServiceDesc = grpc.ServiceDesc{
|
|||||||
MethodName: "RecordDispatch",
|
MethodName: "RecordDispatch",
|
||||||
Handler: _StateStoreService_RecordDispatch_Handler,
|
Handler: _StateStoreService_RecordDispatch_Handler,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
MethodName: "SetDispatchTask",
|
|
||||||
Handler: _StateStoreService_SetDispatchTask_Handler,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
MethodName: "RecordPlaybookDispatch",
|
MethodName: "RecordPlaybookDispatch",
|
||||||
Handler: _StateStoreService_RecordPlaybookDispatch_Handler,
|
Handler: _StateStoreService_RecordPlaybookDispatch_Handler,
|
||||||
|
|||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package daemon
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/config"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/binding"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/mapping"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||||
|
"github.com/samcharles93/archie-core/internal/events"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ledgerStore models the State Store's dispatch ledger: EnqueueBindingTask
|
||||||
|
// claims and creates as one unit, and any ledger write the daemon makes for
|
||||||
|
// a capture refuses a later claim of it, as the real unique index does.
|
||||||
|
type ledgerStore struct {
|
||||||
|
storecontract.TaskStore
|
||||||
|
storecontract.BindingStore
|
||||||
|
storecontract.MappingStore
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
claimed map[string]bool
|
||||||
|
tasks int
|
||||||
|
markers int
|
||||||
|
failures int // enqueues to fail before any commits
|
||||||
|
listed *sync.WaitGroup // holds listers until every admitter has listed
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) ListBindings(context.Context) ([]binding.Binding, error) {
|
||||||
|
return []binding.Binding{dispatchBinding}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) ArmedBindingsForSource(context.Context, string) ([]binding.Binding, error) {
|
||||||
|
return []binding.Binding{dispatchBinding}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) ListUndispatchedCaptures(context.Context, []string, int) ([]storecontract.CapturedEvent, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
pending := !s.claimed["capture"]
|
||||||
|
s.mu.Unlock()
|
||||||
|
if s.listed != nil {
|
||||||
|
s.listed.Done()
|
||||||
|
s.listed.Wait()
|
||||||
|
}
|
||||||
|
if !pending {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return []storecontract.CapturedEvent{{ID: "capture", Source: "forge", EventType: "push", Body: "{}", Authenticated: true}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) RecordDispatch(_ context.Context, _ string, _ int64, captureID string, _ int64, _ string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.claimed[captureID] {
|
||||||
|
return storecontract.ErrAlreadyDispatched
|
||||||
|
}
|
||||||
|
s.claimed[captureID] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) EnqueueBindingTask(_ context.Context, _, _, _, _, _, _, _ string, _ int, captureID string, _ map[string]any) (*workflow.Task, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.claimed[captureID] {
|
||||||
|
return nil, storecontract.ErrAlreadyDispatched
|
||||||
|
}
|
||||||
|
if s.failures > 0 {
|
||||||
|
s.failures--
|
||||||
|
return nil, errors.New("state store unavailable")
|
||||||
|
}
|
||||||
|
s.claimed[captureID] = true
|
||||||
|
s.tasks++
|
||||||
|
return &workflow.Task{ID: int64(s.tasks)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) GetMapping(context.Context, string) (*mapping.Mapping, error) {
|
||||||
|
return &mapping.Mapping{ID: "push", EventTypeID: "push"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ledgerStore) InsertEvent(context.Context, events.Event) (int64, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.markers++
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var dispatchBinding = binding.Binding{
|
||||||
|
ID: "on-push", Name: "on-push", Version: 1, MappingID: "push", Matcher: binding.Matcher{Source: "forge"},
|
||||||
|
OrgID: org.DefaultOrgID, Workflow: "deploy", Status: binding.StatusArmed,
|
||||||
|
}
|
||||||
|
|
||||||
|
// A delivery dispatches exactly one task however its admissions go: replayed
|
||||||
|
// cycles, a failure after the claim, or concurrent dispatchers.
|
||||||
|
func TestBindingDispatchCreatesOneTask(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
failures int
|
||||||
|
cycles int
|
||||||
|
concurrent int
|
||||||
|
}{
|
||||||
|
{name: "replay", cycles: 3, concurrent: 1},
|
||||||
|
{name: "failure after the claim", failures: 1, cycles: 2, concurrent: 1},
|
||||||
|
{name: "concurrent admission", cycles: 1, concurrent: 4},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
store := &ledgerStore{claimed: map[string]bool{}, failures: tc.failures}
|
||||||
|
d := &Daemon{
|
||||||
|
Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler),
|
||||||
|
Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store,
|
||||||
|
WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n"},
|
||||||
|
}
|
||||||
|
for range tc.cycles {
|
||||||
|
var listed, done sync.WaitGroup
|
||||||
|
listed.Add(tc.concurrent)
|
||||||
|
store.listed = &listed
|
||||||
|
for range tc.concurrent {
|
||||||
|
done.Go(func() { d.dispatchBindings(t.Context()) })
|
||||||
|
}
|
||||||
|
done.Wait()
|
||||||
|
}
|
||||||
|
if store.tasks != 1 || store.markers != 1 {
|
||||||
|
t.Fatalf("tasks %d, start markers %d; want exactly one of each", store.tasks, store.markers)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -672,8 +672,8 @@ func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storec
|
|||||||
// dispatchOneBinding offers one capture to one binding: the mapping must
|
// dispatchOneBinding offers one capture to one binding: the mapping must
|
||||||
// match the capture's event type and resolve required fields, the filter
|
// match the capture's event type and resolve required fields, the filter
|
||||||
// must admit the values, and the workflow must accept the inputs. The
|
// must admit the values, and the workflow must accept the inputs. The
|
||||||
// dispatch is claimed in the ledger before the task is enqueued, so each
|
// ledger claim and the task commit together, so each binding fires exactly
|
||||||
// binding fires at most once per capture.
|
// once per delivery.
|
||||||
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
|
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
|
||||||
m := d.bindingMapping(ctx, b, c)
|
m := d.bindingMapping(ctx, b, c)
|
||||||
if m == nil || m.EventTypeID != c.EventType {
|
if m == nil || m.EventTypeID != c.EventType {
|
||||||
@@ -714,16 +714,10 @@ func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c st
|
|||||||
d.claimAndEnqueue(ctx, b, c, target, values)
|
d.claimAndEnqueue(ctx, b, c, target, values)
|
||||||
}
|
}
|
||||||
|
|
||||||
// claimAndEnqueue claims the dispatch in the at-most-once ledger and enqueues
|
// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its
|
||||||
// the task. A failed enqueue after the claim loses that dispatch, which is
|
// task as one unit. A failure leaves the delivery undispatched, so the next
|
||||||
// the at-most-once side of the trade.
|
// cycle offers it again; a delivery already dispatched is refused.
|
||||||
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
|
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
|
||||||
if err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, ""); err != nil {
|
|
||||||
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
|
|
||||||
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
|
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
|
||||||
body := renderBindingBody(values, c)
|
body := renderBindingBody(values, c)
|
||||||
if target.declared {
|
if target.declared {
|
||||||
@@ -733,14 +727,13 @@ func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c store
|
|||||||
if principal, ok := access.PrincipalFromContext(ctx); ok && (principal.IdentityID != d.RootIdentityID || principal.Org != org.DefaultOrgID) {
|
if principal, ok := access.PrincipalFromContext(ctx); ok && (principal.IdentityID != d.RootIdentityID || principal.Org != org.DefaultOrgID) {
|
||||||
taskIdentity = string(principal.IdentityID)
|
taskIdentity = string(principal.IdentityID)
|
||||||
}
|
}
|
||||||
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, target.inputs)
|
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
|
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
|
||||||
|
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := d.BindingDispatcher.SetDispatchTask(ctx, b.ID, c.ID, task.ID); err != nil {
|
|
||||||
d.Log.Warn("binding dispatch: record task", "binding", b.ID, "capture", c.ID, "task", task.ID, "error", err)
|
|
||||||
}
|
|
||||||
d.markBindingStart(ctx, task.ID, b, c)
|
d.markBindingStart(ctx, task.ID, b, c)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -57,14 +57,13 @@ func (*orgBindingStore) ListUndispatchedCaptures(_ context.Context, sources []st
|
|||||||
func (*orgBindingStore) RecordDispatch(context.Context, string, int64, string, int64, string) error {
|
func (*orgBindingStore) RecordDispatch(context.Context, string, int64, string, int64, string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (*orgBindingStore) SetDispatchTask(context.Context, string, string, int64) error { return nil }
|
func (*orgBindingStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
||||||
func (*orgBindingStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
|
|
||||||
|
|
||||||
func (*orgBindingStore) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) {
|
func (*orgBindingStore) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) {
|
||||||
return &mapping.Mapping{EventTypeID: id}, nil
|
return &mapping.Mapping{EventTypeID: id}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *orgBindingStore) EnqueueBindingTask(ctx context.Context, owner, repo, _, _, wf, identity, bindingID string, _ int, _ map[string]any) (*workflow.Task, error) {
|
func (s *orgBindingStore) EnqueueBindingTask(ctx context.Context, owner, repo, _, _, wf, identity, bindingID string, _ int, _ string, _ map[string]any) (*workflow.Task, error) {
|
||||||
run := &workflow.Task{ID: 1, Owner: owner, Repo: repo, Workflow: wf, Identity: identity, Org: org.OrgFromContext(ctx), BindingID: bindingID}
|
run := &workflow.Task{ID: 1, Owner: owner, Repo: repo, Workflow: wf, Identity: identity, Org: org.OrgFromContext(ctx), BindingID: bindingID}
|
||||||
s.started[run.Org] = run
|
s.started[run.Org] = run
|
||||||
return run, nil
|
return run, nil
|
||||||
|
|||||||
@@ -320,16 +320,14 @@ type BindingDispatcher interface {
|
|||||||
bindingVersion int64,
|
bindingVersion int64,
|
||||||
captureID string,
|
captureID string,
|
||||||
// taskID addresses the SQLite-owned task tables, which are not
|
// taskID addresses the SQLite-owned task tables, which are not
|
||||||
// migrating: it stays an integer on purpose. The binding dispatch
|
// migrating: it stays an integer on purpose. A dispatch that starts
|
||||||
// loop claims before it enqueues, so it records 0.
|
// a task is recorded by EnqueueBindingTask instead.
|
||||||
taskID int64,
|
taskID int64,
|
||||||
// reason is empty for a dispatch and names the terminal outcome of
|
// reason is empty for a dispatch and names the terminal outcome of
|
||||||
// an evaluation that started no task.
|
// an evaluation that started no task.
|
||||||
reason string,
|
reason string,
|
||||||
) error
|
) error
|
||||||
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
|
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
|
||||||
// SetDispatchTask records the task a claimed dispatch started.
|
|
||||||
SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BindingTaskCreator is the single-method consumer-facing surface for
|
// BindingTaskCreator is the single-method consumer-facing surface for
|
||||||
@@ -337,7 +335,11 @@ type BindingDispatcher interface {
|
|||||||
// keeps the lifecycle surface narrow (8 methods, the interfacebloat limit)
|
// keeps the lifecycle surface narrow (8 methods, the interfacebloat limit)
|
||||||
// and keeps the binding-specific shape on the binding interfaces.
|
// and keeps the binding-specific shape on the binding interfaces.
|
||||||
type BindingTaskCreator interface {
|
type BindingTaskCreator interface {
|
||||||
EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*task.Task, error)
|
// EnqueueBindingTask claims the binding's dispatch of captureID in the
|
||||||
|
// ledger, enqueues the task and records its provenance as one unit: an
|
||||||
|
// error leaves none of them, and a delivery already dispatched for the
|
||||||
|
// binding returns ErrAlreadyDispatched.
|
||||||
|
EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*task.Task, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WorkflowCaller starts a workflow.call callee and reads it back.
|
// WorkflowCaller starts a workflow.call callee and reads it back.
|
||||||
|
|||||||
@@ -550,15 +550,6 @@ func (s *EDA) RecordDispatch(ctx context.Context, bindingID string, bindingVersi
|
|||||||
return ledgerWrite(err, "edastore: record dispatch")
|
return ledgerWrite(err, "edastore: record dispatch")
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetDispatchTask records the task a claimed dispatch started.
|
|
||||||
func (s *EDA) SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error {
|
|
||||||
err := s.q.SetBindingDispatchTask(ctx, postgresdb.SetBindingDispatchTaskParams{Binding: bindingID, Capture: captureID, TaskID: taskID})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("edastore: set dispatch task: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ListDispatches returns the newest ledger rows matching filter.
|
// ListDispatches returns the newest ledger rows matching filter.
|
||||||
func (s *EDA) ListDispatches(ctx context.Context, filter storecontract.DispatchFilter) ([]storecontract.Dispatch, error) {
|
func (s *EDA) ListDispatches(ctx context.Context, filter storecontract.DispatchFilter) ([]storecontract.Dispatch, error) {
|
||||||
params := postgresdb.ListBindingDispatchesParams{EntryLimit: int32(min(max(filter.Limit, 1), 500)), ScopeOrg: scopeOrg(ctx)} // clamped
|
params := postgresdb.ListBindingDispatchesParams{EntryLimit: int32(min(max(filter.Limit, 1), 500)), ScopeOrg: scopeOrg(ctx)} // clamped
|
||||||
|
|||||||
@@ -156,11 +156,15 @@ func (s *Store) EnqueueIssue(ctx context.Context, owner, repo string, number int
|
|||||||
// EnqueueChatTask inserts a queued chat task with a store-allocated synthetic
|
// EnqueueChatTask inserts a queued chat task with a store-allocated synthetic
|
||||||
// issue number. origin names the conversation that created it, if any.
|
// issue number. origin names the conversation that created it, if any.
|
||||||
func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, wf, identity, origin string, inputs map[string]any) (*workflow.Task, error) {
|
func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, wf, identity, origin string, inputs map[string]any) (*workflow.Task, error) {
|
||||||
|
return insertChatTask(ctx, s.queries(), owner, repo, title, body, wf, identity, origin, inputs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func insertChatTask(ctx context.Context, q *postgresdb.Queries, owner, repo, title, body, wf, identity, origin string, inputs map[string]any) (*workflow.Task, error) {
|
||||||
encoded, err := task.EncodeInputs(inputs)
|
encoded, err := task.EncodeInputs(inputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
t, err := s.queries().InsertChatTask(ctx, postgresdb.InsertChatTaskParams{
|
t, err := q.InsertChatTask(ctx, postgresdb.InsertChatTaskParams{
|
||||||
ScopeOrg: scopeOrg(ctx),
|
ScopeOrg: scopeOrg(ctx),
|
||||||
Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity,
|
Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity,
|
||||||
Inputs: encoded,
|
Inputs: encoded,
|
||||||
@@ -176,20 +180,46 @@ func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, w
|
|||||||
return taskFromRow(t), nil
|
return taskFromRow(t), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnqueueBindingTask enqueues a binding-triggered task and stamps its binding
|
// EnqueueBindingTask claims the dispatch ledger row for the capture's
|
||||||
// provenance in a second statement (best-effort, as in the SQLite store). The
|
// delivery, inserts the task, stamps its binding provenance and records the
|
||||||
// inputs travel with the insert, so a failed stamp leaves a task whose inputs
|
// task on the ledger row in one transaction. A concurrent claim of the same
|
||||||
// are already right and whose provenance is absent, never the reverse.
|
// delivery waits on the ledger's unique index and is refused once this one
|
||||||
func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*workflow.Task, error) {
|
// commits, so a replay creates no second task, and a failure anywhere leaves
|
||||||
t, err := s.EnqueueChatTask(ctx, owner, repo, title, body, wf, identity, "", inputs)
|
// the delivery undispatched for the next cycle.
|
||||||
|
func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*workflow.Task, error) {
|
||||||
|
tx, err := s.pool.Begin(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := s.queries().StampTaskBinding(ctx, postgresdb.StampTaskBindingParams{
|
defer func() { _ = tx.Rollback(ctx) }()
|
||||||
|
q := postgresdb.New(tx)
|
||||||
|
|
||||||
|
n, err := q.InsertBindingDispatch(ctx, postgresdb.InsertBindingDispatchParams{
|
||||||
|
Binding: bindingID, BindingVersion: int64(bindingVersion), Capture: captureID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, ledgerWrite(err, "store: claim binding dispatch")
|
||||||
|
}
|
||||||
|
if n == 0 {
|
||||||
|
return nil, fmt.Errorf("store: claim binding dispatch: capture %q not found", captureID)
|
||||||
|
}
|
||||||
|
t, err := insertChatTask(ctx, q, owner, repo, title, body, wf, identity, "", inputs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := q.StampTaskBinding(ctx, postgresdb.StampTaskBindingParams{
|
||||||
ID: t.ID, BindingID: bindingID, BindingVersion: int64(bindingVersion),
|
ID: t.ID, BindingID: bindingID, BindingVersion: int64(bindingVersion),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, fmt.Errorf("store: stamp binding provenance: %w", err)
|
return nil, fmt.Errorf("store: stamp binding provenance: %w", err)
|
||||||
}
|
}
|
||||||
|
if err := q.SetBindingDispatchTask(ctx, postgresdb.SetBindingDispatchTaskParams{
|
||||||
|
Binding: bindingID, Capture: captureID, TaskID: t.ID,
|
||||||
|
}); err != nil {
|
||||||
|
return nil, fmt.Errorf("store: record dispatch task: %w", err)
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
t.BindingID = bindingID
|
t.BindingID = bindingID
|
||||||
t.BindingVersion = bindingVersion
|
t.BindingVersion = bindingVersion
|
||||||
return t, nil
|
return t, nil
|
||||||
|
|||||||
@@ -649,11 +649,6 @@ func (c *Client) ListDispatches(ctx context.Context, filter storecontract.Dispat
|
|||||||
}), nil
|
}), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error {
|
|
||||||
_, err := c.client.SetDispatchTask(ctx, &pb.SetDispatchTaskRequest{BindingId: bindingID, CaptureId: captureID, TaskId: taskID})
|
|
||||||
return unmapError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Client) UpdateBinding(ctx context.Context, b binding.Binding) error {
|
func (c *Client) UpdateBinding(ctx context.Context, b binding.Binding) error {
|
||||||
_, err := c.client.UpdateBinding(ctx, &pb.UpdateBindingRequest{Binding: bindingProto(b)})
|
_, err := c.client.UpdateBinding(ctx, &pb.UpdateBindingRequest{Binding: bindingProto(b)})
|
||||||
return unmapError(err)
|
return unmapError(err)
|
||||||
@@ -779,12 +774,12 @@ func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string,
|
|||||||
|
|
||||||
// BindingTaskCreator
|
// BindingTaskCreator
|
||||||
|
|
||||||
func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*task.Task, error) {
|
func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*task.Task, error) {
|
||||||
encoded, err := task.EncodeInputs(inputs)
|
encoded, err := task.EncodeInputs(inputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
r, err := c.client.EnqueueBindingTask(ctx, &pb.EnqueueBindingTaskRequest{Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity, BindingId: bindingID, BindingVersion: int64(bindingVersion), InputsJson: encoded})
|
r, err := c.client.EnqueueBindingTask(ctx, &pb.EnqueueBindingTaskRequest{Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity, BindingId: bindingID, BindingVersion: int64(bindingVersion), InputsJson: encoded, CaptureId: captureID})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, unmapError(err)
|
return nil, unmapError(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1114,16 +1114,6 @@ func (s *server) RecordDispatch(ctx context.Context, r *pb.RecordDispatchRequest
|
|||||||
return &pb.RecordDispatchResponse{}, nil
|
return &pb.RecordDispatchResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *server) SetDispatchTask(ctx context.Context, r *pb.SetDispatchTaskRequest) (*pb.SetDispatchTaskResponse, error) {
|
|
||||||
if s.deps.BindingDispatcher == nil {
|
|
||||||
return nil, errBindingDispatchUnavailable
|
|
||||||
}
|
|
||||||
if err := s.deps.BindingDispatcher.SetDispatchTask(ctx, r.BindingId, r.CaptureId, r.TaskId); err != nil {
|
|
||||||
return nil, s.logErr("SetDispatchTask", err)
|
|
||||||
}
|
|
||||||
return &pb.SetDispatchTaskResponse{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *server) ListDispatches(ctx context.Context, r *pb.ListDispatchesRequest) (*pb.ListDispatchesResponse, error) {
|
func (s *server) ListDispatches(ctx context.Context, r *pb.ListDispatchesRequest) (*pb.ListDispatchesResponse, error) {
|
||||||
bs, err := s.binding()
|
bs, err := s.binding()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1196,7 +1186,7 @@ func (s *server) EnqueueBindingTask(ctx context.Context, r *pb.EnqueueBindingTas
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||||
}
|
}
|
||||||
t, err := s.deps.BindingTaskCreator.EnqueueBindingTask(ctx, r.Owner, r.Repo, r.Title, r.Body, r.Workflow, r.Identity, r.BindingId, int(r.BindingVersion), inputs)
|
t, err := s.deps.BindingTaskCreator.EnqueueBindingTask(ctx, r.Owner, r.Repo, r.Title, r.Body, r.Workflow, r.Identity, r.BindingId, int(r.BindingVersion), r.CaptureId, inputs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, s.logErr("EnqueueBindingTask", err)
|
return nil, s.logErr("EnqueueBindingTask", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -230,7 +230,6 @@ service StateStoreService {
|
|||||||
// Dispatch
|
// Dispatch
|
||||||
rpc ArmedBindingsForSource(ArmedBindingsForSourceRequest) returns (ArmedBindingsForSourceResponse);
|
rpc ArmedBindingsForSource(ArmedBindingsForSourceRequest) returns (ArmedBindingsForSourceResponse);
|
||||||
rpc RecordDispatch(RecordDispatchRequest) returns (RecordDispatchResponse);
|
rpc RecordDispatch(RecordDispatchRequest) returns (RecordDispatchResponse);
|
||||||
rpc SetDispatchTask(SetDispatchTaskRequest) returns (SetDispatchTaskResponse);
|
|
||||||
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
|
||||||
// at-most-once record per (playbook, version, event, action) for the
|
// at-most-once record per (playbook, version, event, action) for the
|
||||||
// side-effecting positions. Administrative, like RecordDispatch: a
|
// side-effecting positions. Administrative, like RecordDispatch: a
|
||||||
@@ -239,7 +238,9 @@ service StateStoreService {
|
|||||||
rpc DeletePlaybookDispatches(DeletePlaybookDispatchesRequest) returns (DeletePlaybookDispatchesResponse);
|
rpc DeletePlaybookDispatches(DeletePlaybookDispatchesRequest) returns (DeletePlaybookDispatchesResponse);
|
||||||
rpc StreamUndispatchedCaptures(StreamUndispatchedCapturesRequest) returns (stream StreamUndispatchedCapturesResponse);
|
rpc StreamUndispatchedCaptures(StreamUndispatchedCapturesRequest) returns (stream StreamUndispatchedCapturesResponse);
|
||||||
|
|
||||||
// BindingTaskCreator
|
// BindingTaskCreator: claims the binding's dispatch of a capture and
|
||||||
|
// enqueues its task in one transaction, so a failure leaves neither and a
|
||||||
|
// replayed delivery is refused with AlreadyExists.
|
||||||
rpc EnqueueBindingTask(EnqueueBindingTaskRequest) returns (EnqueueBindingTaskResponse);
|
rpc EnqueueBindingTask(EnqueueBindingTaskRequest) returns (EnqueueBindingTaskResponse);
|
||||||
|
|
||||||
// WorkflowCaller: a workflow.call step starts its callee and, with
|
// WorkflowCaller: a workflow.call step starts its callee and, with
|
||||||
@@ -1246,13 +1247,6 @@ message RecordDispatchRequest {
|
|||||||
}
|
}
|
||||||
message RecordDispatchResponse {}
|
message RecordDispatchResponse {}
|
||||||
|
|
||||||
message SetDispatchTaskRequest {
|
|
||||||
string binding_id = 1;
|
|
||||||
string capture_id = 2;
|
|
||||||
int64 task_id = 3;
|
|
||||||
}
|
|
||||||
message SetDispatchTaskResponse {}
|
|
||||||
|
|
||||||
// ListDispatchesRequest filters the binding dispatch ledger; empty fields
|
// ListDispatchesRequest filters the binding dispatch ledger; empty fields
|
||||||
// match any row.
|
// match any row.
|
||||||
message ListDispatchesRequest {
|
message ListDispatchesRequest {
|
||||||
@@ -1299,6 +1293,7 @@ message EnqueueBindingTaskRequest {
|
|||||||
string binding_id = 7;
|
string binding_id = 7;
|
||||||
int64 binding_version = 8;
|
int64 binding_version = 8;
|
||||||
string inputs_json = 9;
|
string inputs_json = 9;
|
||||||
|
string capture_id = 10;
|
||||||
}
|
}
|
||||||
message EnqueueBindingTaskResponse { Task task = 1; }
|
message EnqueueBindingTaskResponse { Task task = 1; }
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user