Merge pull request #1601 from samcharles93/fix/atomic-dispatch

Fix/atomic dispatch
This commit is contained in:
sam authored and GitHub committed 2026-10-12 01:29:16 +11:00
commit d66dca44e0
17 files changed
+911 -886

No files matched your search

File diff suppressed because it is too large. Load diff
+6 -40
View File
@@ -130,7 +130,6 @@ const (
StateStoreService_DeleteSource_FullMethodName = "/state.v1.StateStoreService/DeleteSource"
StateStoreService_ArmedBindingsForSource_FullMethodName = "/state.v1.StateStoreService/ArmedBindingsForSource"
StateStoreService_RecordDispatch_FullMethodName = "/state.v1.StateStoreService/RecordDispatch"
StateStoreService_SetDispatchTask_FullMethodName = "/state.v1.StateStoreService/SetDispatchTask"
StateStoreService_RecordPlaybookDispatch_FullMethodName = "/state.v1.StateStoreService/RecordPlaybookDispatch"
StateStoreService_DeletePlaybookDispatches_FullMethodName = "/state.v1.StateStoreService/DeletePlaybookDispatches"
StateStoreService_StreamUndispatchedCaptures_FullMethodName = "/state.v1.StateStoreService/StreamUndispatchedCaptures"
@@ -369,7 +368,6 @@ type StateStoreServiceClient interface {
// Dispatch
ArmedBindingsForSource(ctx context.Context, in *ArmedBindingsForSourceRequest, opts ...grpc.CallOption) (*ArmedBindingsForSourceResponse, error)
RecordDispatch(ctx context.Context, in *RecordDispatchRequest, opts ...grpc.CallOption) (*RecordDispatchResponse, error)
SetDispatchTask(ctx context.Context, in *SetDispatchTaskRequest, opts ...grpc.CallOption) (*SetDispatchTaskResponse, error)
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
// at-most-once record per (playbook, version, event, action) for the
// side-effecting positions. Administrative, like RecordDispatch: a
@@ -377,7 +375,9 @@ type StateStoreServiceClient interface {
RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error)
DeletePlaybookDispatches(ctx context.Context, in *DeletePlaybookDispatchesRequest, opts ...grpc.CallOption) (*DeletePlaybookDispatchesResponse, error)
StreamUndispatchedCaptures(ctx context.Context, in *StreamUndispatchedCapturesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamUndispatchedCapturesResponse], error)
// BindingTaskCreator
// BindingTaskCreator: claims the binding's dispatch of a capture and
// enqueues its task in one transaction, so a failure leaves neither and a
// replayed delivery is refused with AlreadyExists.
EnqueueBindingTask(ctx context.Context, in *EnqueueBindingTaskRequest, opts ...grpc.CallOption) (*EnqueueBindingTaskResponse, error)
// WorkflowCaller: a workflow.call step starts its callee and, with
// wait:true, reads it back while waiting. Both are the one sanctioned
@@ -1551,16 +1551,6 @@ func (c *stateStoreServiceClient) RecordDispatch(ctx context.Context, in *Record
return out, nil
}
func (c *stateStoreServiceClient) SetDispatchTask(ctx context.Context, in *SetDispatchTaskRequest, opts ...grpc.CallOption) (*SetDispatchTaskResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(SetDispatchTaskResponse)
err := c.cc.Invoke(ctx, StateStoreService_SetDispatchTask_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *stateStoreServiceClient) RecordPlaybookDispatch(ctx context.Context, in *RecordPlaybookDispatchRequest, opts ...grpc.CallOption) (*RecordPlaybookDispatchResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(RecordPlaybookDispatchResponse)
@@ -2004,7 +1994,6 @@ type StateStoreServiceServer interface {
// Dispatch
ArmedBindingsForSource(context.Context, *ArmedBindingsForSourceRequest) (*ArmedBindingsForSourceResponse, error)
RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error)
SetDispatchTask(context.Context, *SetDispatchTaskRequest) (*SetDispatchTaskResponse, error)
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
// at-most-once record per (playbook, version, event, action) for the
// side-effecting positions. Administrative, like RecordDispatch: a
@@ -2012,7 +2001,9 @@ type StateStoreServiceServer interface {
RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error)
DeletePlaybookDispatches(context.Context, *DeletePlaybookDispatchesRequest) (*DeletePlaybookDispatchesResponse, error)
StreamUndispatchedCaptures(*StreamUndispatchedCapturesRequest, grpc.ServerStreamingServer[StreamUndispatchedCapturesResponse]) error
// BindingTaskCreator
// BindingTaskCreator: claims the binding's dispatch of a capture and
// enqueues its task in one transaction, so a failure leaves neither and a
// replayed delivery is refused with AlreadyExists.
EnqueueBindingTask(context.Context, *EnqueueBindingTaskRequest) (*EnqueueBindingTaskResponse, error)
// WorkflowCaller: a workflow.call step starts its callee and, with
// wait:true, reads it back while waiting. Both are the one sanctioned
@@ -2391,9 +2382,6 @@ func (UnimplementedStateStoreServiceServer) ArmedBindingsForSource(context.Conte
func (UnimplementedStateStoreServiceServer) RecordDispatch(context.Context, *RecordDispatchRequest) (*RecordDispatchResponse, error) {
return nil, status.Error(codes.Unimplemented, "method RecordDispatch not implemented")
}
func (UnimplementedStateStoreServiceServer) SetDispatchTask(context.Context, *SetDispatchTaskRequest) (*SetDispatchTaskResponse, error) {
return nil, status.Error(codes.Unimplemented, "method SetDispatchTask not implemented")
}
func (UnimplementedStateStoreServiceServer) RecordPlaybookDispatch(context.Context, *RecordPlaybookDispatchRequest) (*RecordPlaybookDispatchResponse, error) {
return nil, status.Error(codes.Unimplemented, "method RecordPlaybookDispatch not implemented")
}
@@ -4465,24 +4453,6 @@ func _StateStoreService_RecordDispatch_Handler(srv interface{}, ctx context.Cont
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_SetDispatchTask_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SetDispatchTaskRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(StateStoreServiceServer).SetDispatchTask(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: StateStoreService_SetDispatchTask_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(StateStoreServiceServer).SetDispatchTask(ctx, req.(*SetDispatchTaskRequest))
}
return interceptor(ctx, in, info, handler)
}
func _StateStoreService_RecordPlaybookDispatch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RecordPlaybookDispatchRequest)
if err := dec(in); err != nil {
@@ -5315,10 +5285,6 @@ var StateStoreService_ServiceDesc = grpc.ServiceDesc{
MethodName: "RecordDispatch",
Handler: _StateStoreService_RecordDispatch_Handler,
},
{
MethodName: "SetDispatchTask",
Handler: _StateStoreService_SetDispatchTask_Handler,
},
{
MethodName: "RecordPlaybookDispatch",
Handler: _StateStoreService_RecordPlaybookDispatch_Handler,
+132
View File
@@ -0,0 +1,132 @@
package daemon
import (
"context"
"errors"
"log/slog"
"sync"
"testing"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/binding"
"github.com/samcharles93/archie-core/internal/domain/mapping"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/events"
)
// ledgerStore models the State Store's dispatch ledger: EnqueueBindingTask
// claims and creates as one unit, and any ledger write the daemon makes for
// a capture refuses a later claim of it, as the real unique index does.
type ledgerStore struct {
storecontract.TaskStore
storecontract.BindingStore
storecontract.MappingStore
mu sync.Mutex
claimed map[string]bool
tasks int
markers int
failures int // enqueues to fail before any commits
listed *sync.WaitGroup // holds listers until every admitter has listed
}
func (s *ledgerStore) ListBindings(context.Context) ([]binding.Binding, error) {
return []binding.Binding{dispatchBinding}, nil
}
func (s *ledgerStore) ArmedBindingsForSource(context.Context, string) ([]binding.Binding, error) {
return []binding.Binding{dispatchBinding}, nil
}
func (s *ledgerStore) ListUndispatchedCaptures(context.Context, []string, int) ([]storecontract.CapturedEvent, error) {
s.mu.Lock()
pending := !s.claimed["capture"]
s.mu.Unlock()
if s.listed != nil {
s.listed.Done()
s.listed.Wait()
}
if !pending {
return nil, nil
}
return []storecontract.CapturedEvent{{ID: "capture", Source: "forge", EventType: "push", Body: "{}", Authenticated: true}}, nil
}
func (s *ledgerStore) RecordDispatch(_ context.Context, _ string, _ int64, captureID string, _ int64, _ string) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.claimed[captureID] {
return storecontract.ErrAlreadyDispatched
}
s.claimed[captureID] = true
return nil
}
func (s *ledgerStore) EnqueueBindingTask(_ context.Context, _, _, _, _, _, _, _ string, _ int, captureID string, _ map[string]any) (*workflow.Task, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.claimed[captureID] {
return nil, storecontract.ErrAlreadyDispatched
}
if s.failures > 0 {
s.failures--
return nil, errors.New("state store unavailable")
}
s.claimed[captureID] = true
s.tasks++
return &workflow.Task{ID: int64(s.tasks)}, nil
}
func (s *ledgerStore) GetMapping(context.Context, string) (*mapping.Mapping, error) {
return &mapping.Mapping{ID: "push", EventTypeID: "push"}, nil
}
func (s *ledgerStore) InsertEvent(context.Context, events.Event) (int64, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.markers++
return 0, nil
}
var dispatchBinding = binding.Binding{
ID: "on-push", Name: "on-push", Version: 1, MappingID: "push", Matcher: binding.Matcher{Source: "forge"},
OrgID: org.DefaultOrgID, Workflow: "deploy", Status: binding.StatusArmed,
}
// A delivery dispatches exactly one task however its admissions go: replayed
// cycles, a failure after the claim, or concurrent dispatchers.
func TestBindingDispatchCreatesOneTask(t *testing.T) {
for _, tc := range []struct {
name string
failures int
cycles int
concurrent int
}{
{name: "replay", cycles: 3, concurrent: 1},
{name: "failure after the claim", failures: 1, cycles: 2, concurrent: 1},
{name: "concurrent admission", cycles: 1, concurrent: 4},
} {
t.Run(tc.name, func(t *testing.T) {
store := &ledgerStore{claimed: map[string]bool{}, failures: tc.failures}
d := &Daemon{
Cfg: config.NewHolder(config.Config{}), Store: store, Log: slog.New(slog.DiscardHandler),
Bindings: store, BindingDispatcher: store, BindingTaskCreator: store, Mappings: store,
WorkflowDefinitions: orgDefinitions{org.DefaultOrgID: "id: deploy\nrepository: none\nsteps:\n - type: workflow.finish\n"},
}
for range tc.cycles {
var listed, done sync.WaitGroup
listed.Add(tc.concurrent)
store.listed = &listed
for range tc.concurrent {
done.Go(func() { d.dispatchBindings(t.Context()) })
}
done.Wait()
}
if store.tasks != 1 || store.markers != 1 {
t.Fatalf("tasks %d, start markers %d; want exactly one of each", store.tasks, store.markers)
}
})
}
}
+9 -16
View File
@@ -672,8 +672,8 @@ func (d *Daemon) bindingMapping(ctx context.Context, b binding.Binding, c storec
// dispatchOneBinding offers one capture to one binding: the mapping must
// match the capture's event type and resolve required fields, the filter
// must admit the values, and the workflow must accept the inputs. The
// dispatch is claimed in the ledger before the task is enqueued, so each
// binding fires at most once per capture.
// ledger claim and the task commit together, so each binding fires exactly
// once per delivery.
func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, workflows workflow.WorkflowDefinitionCollection) {
m := d.bindingMapping(ctx, b, c)
if m == nil || m.EventTypeID != c.EventType {
@@ -714,16 +714,10 @@ func (d *Daemon) dispatchOneBinding(ctx context.Context, b binding.Binding, c st
d.claimAndEnqueue(ctx, b, c, target, values)
}
// claimAndEnqueue claims the dispatch in the at-most-once ledger and enqueues
// the task. A failed enqueue after the claim loses that dispatch, which is
// the at-most-once side of the trade.
// claimAndEnqueue asks the State Store to claim the dispatch and enqueue its
// task as one unit. A failure leaves the delivery undispatched, so the next
// cycle offers it again; a delivery already dispatched is refused.
func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c storecontract.CapturedEvent, target bindingTarget, values map[string]any) {
if err := d.BindingDispatcher.RecordDispatch(ctx, b.ID, int64(b.Version), c.ID, 0, ""); err != nil {
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.Warn("binding dispatch: record", "binding", b.ID, "capture", c.ID, "error", err)
}
return
}
title := fmt.Sprintf("binding %s/%d from %s", b.Name, b.Version, c.Source)
body := renderBindingBody(values, c)
if target.declared {
@@ -733,14 +727,13 @@ func (d *Daemon) claimAndEnqueue(ctx context.Context, b binding.Binding, c store
if principal, ok := access.PrincipalFromContext(ctx); ok && (principal.IdentityID != d.RootIdentityID || principal.Org != org.DefaultOrgID) {
taskIdentity = string(principal.IdentityID)
}
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, target.inputs)
task, err := d.BindingTaskCreator.EnqueueBindingTask(ctx, target.owner, target.repo, title, body, b.Workflow, taskIdentity, b.ID, b.Version, c.ID, target.inputs)
if err != nil {
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
if !errors.Is(err, storecontract.ErrAlreadyDispatched) {
d.Log.Warn("binding dispatch: enqueue", "binding", b.ID, "capture", c.ID, "error", err)
}
return
}
if err := d.BindingDispatcher.SetDispatchTask(ctx, b.ID, c.ID, task.ID); err != nil {
d.Log.Warn("binding dispatch: record task", "binding", b.ID, "capture", c.ID, "task", task.ID, "error", err)
}
d.markBindingStart(ctx, task.ID, b, c)
}
+2 -3
View File
@@ -57,14 +57,13 @@ func (*orgBindingStore) ListUndispatchedCaptures(_ context.Context, sources []st
func (*orgBindingStore) RecordDispatch(context.Context, string, int64, string, int64, string) error {
return nil
}
func (*orgBindingStore) SetDispatchTask(context.Context, string, string, int64) error { return nil }
func (*orgBindingStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
func (*orgBindingStore) InsertEvent(context.Context, events.Event) (int64, error) { return 0, nil }
func (*orgBindingStore) GetMapping(_ context.Context, id string) (*mapping.Mapping, error) {
return &mapping.Mapping{EventTypeID: id}, nil
}
func (s *orgBindingStore) EnqueueBindingTask(ctx context.Context, owner, repo, _, _, wf, identity, bindingID string, _ int, _ map[string]any) (*workflow.Task, error) {
func (s *orgBindingStore) EnqueueBindingTask(ctx context.Context, owner, repo, _, _, wf, identity, bindingID string, _ int, _ string, _ map[string]any) (*workflow.Task, error) {
run := &workflow.Task{ID: 1, Owner: owner, Repo: repo, Workflow: wf, Identity: identity, Org: org.OrgFromContext(ctx), BindingID: bindingID}
s.started[run.Org] = run
return run, nil
@@ -320,16 +320,14 @@ type BindingDispatcher interface {
bindingVersion int64,
captureID string,
// taskID addresses the SQLite-owned task tables, which are not
// migrating: it stays an integer on purpose. The binding dispatch
// loop claims before it enqueues, so it records 0.
// migrating: it stays an integer on purpose. A dispatch that starts
// a task is recorded by EnqueueBindingTask instead.
taskID int64,
// reason is empty for a dispatch and names the terminal outcome of
// an evaluation that started no task.
reason string,
) error
ListUndispatchedCaptures(ctx context.Context, sources []string, limit int) ([]CapturedEvent, error)
// SetDispatchTask records the task a claimed dispatch started.
SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error
}
// BindingTaskCreator is the single-method consumer-facing surface for
@@ -337,7 +335,11 @@ type BindingDispatcher interface {
// keeps the lifecycle surface narrow (8 methods, the interfacebloat limit)
// and keeps the binding-specific shape on the binding interfaces.
type BindingTaskCreator interface {
EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*task.Task, error)
// EnqueueBindingTask claims the binding's dispatch of captureID in the
// ledger, enqueues the task and records its provenance as one unit: an
// error leaves none of them, and a delivery already dispatched for the
// binding returns ErrAlreadyDispatched.
EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*task.Task, error)
}
// WorkflowCaller starts a workflow.call callee and reads it back.
@@ -550,15 +550,6 @@ func (s *EDA) RecordDispatch(ctx context.Context, bindingID string, bindingVersi
return ledgerWrite(err, "edastore: record dispatch")
}
// SetDispatchTask records the task a claimed dispatch started.
func (s *EDA) SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error {
err := s.q.SetBindingDispatchTask(ctx, postgresdb.SetBindingDispatchTaskParams{Binding: bindingID, Capture: captureID, TaskID: taskID})
if err != nil {
return fmt.Errorf("edastore: set dispatch task: %w", err)
}
return nil
}
// ListDispatches returns the newest ledger rows matching filter.
func (s *EDA) ListDispatches(ctx context.Context, filter storecontract.DispatchFilter) ([]storecontract.Dispatch, error) {
params := postgresdb.ListBindingDispatchesParams{EntryLimit: int32(min(max(filter.Limit, 1), 500)), ScopeOrg: scopeOrg(ctx)} // clamped
@@ -0,0 +1,19 @@
-- chat_issue_numbers holds the last synthetic issue number allocated in each
-- repository. Chat and workflow.call tasks take theirs by bumping this row,
-- whose row lock serializes concurrent inserts in one repository, which a
-- MAX(issue_number) read under READ COMMITTED cannot.
-- +goose Up
CREATE TABLE chat_issue_numbers (
owner text NOT NULL,
repo text NOT NULL,
last_number bigint NOT NULL,
PRIMARY KEY (owner, repo)
);
INSERT INTO chat_issue_numbers (owner, repo, last_number)
SELECT owner, repo, MAX(issue_number) FROM tasks WHERE source = 'chat' GROUP BY owner, repo;
-- +goose Down
DROP TABLE chat_issue_numbers;
@@ -25,35 +25,35 @@ func (q *Queries) CallStatusDetail(ctx context.Context, taskID int64) (string, e
const enqueueCallTask = `-- name: EnqueueCallTask :one
WITH caller AS (
SELECT t.id, t.owner, t.repo, t.issue_number, t.title, t.body, t.labels, t.status, t.workflow, t.branch, t.plan, t.notes, t.pr_number, t.tokens_used, t.iterations, t.attempt, t.park_reason, t.watch_comment_id, t.park_class, t.remediation_rounds, t.retry_count, t.source, t.identity, t.binding_id, t.binding_version, t.review_payload, t.workflow_definition_version, t.workflow_definition_digest, t.workflow_definition_yaml, t.created_at, t.updated_at, t.review_cursor, t.inputs, t.org_id, t.workspace_id, t.call_parent_task_id, t.call_depth, t.outputs, t.review_gate, t.rereview_rounds, t.retry_mode, t.resume_from, t.resume_results, t.pending_reviews, t.origin, t.call_key, t.outcome_posted FROM tasks t WHERE t.id = $1 FOR UPDATE
), number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number)
SELECT caller.owner, caller.repo, $8::bigint + 1 FROM caller
ON CONFLICT (owner, repo) DO UPDATE SET last_number = chat_issue_numbers.last_number + 1
RETURNING last_number
)
INSERT INTO tasks (owner, repo, issue_number, title, body, labels, workflow, source, identity, org_id, workspace_id, inputs, call_parent_task_id, call_depth, call_key)
SELECT
caller.owner,
caller.repo,
COALESCE((
SELECT MAX(existing.issue_number) FROM tasks existing
WHERE existing.owner = caller.owner
AND existing.repo = caller.repo
AND existing.source = 'chat'
), $2::bigint) + 1,
$3, $4, 'chat', $5, 'chat', caller.identity,
caller.org_id, caller.workspace_id, $6,
caller.id, caller.call_depth + 1, $7
FROM caller
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $8::int
number.last_number,
$2, $3, 'chat', $4, 'chat', caller.identity,
caller.org_id, caller.workspace_id, $5,
caller.id, caller.call_depth + 1, $6
FROM caller, number
WHERE caller.status = 'running' AND caller.call_depth + 1 <= $7::int
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, outcome_posted
`
type EnqueueCallTaskParams struct {
ID int64
FallbackIssueNumber int64
Title string
Body string
Workflow string
Inputs string
CallKey string
MaxDepth int32
FallbackIssueNumber int64
}
// The callee of a workflow.call step (docs/prds/workflow-calls.md)
@@ -64,17 +64,18 @@ type EnqueueCallTaskParams struct {
// child that ended dead or closed_wont_do freed its key -- and the WHERE below
// still decides whether the caller may call at all: a caller that is missing,
// not running or past the depth limit inserts nothing and returns no row,
// conflict or not.
// conflict or not. The issue number comes from chat_issue_numbers, as for
// InsertChatTask.
func (q *Queries) EnqueueCallTask(ctx context.Context, arg EnqueueCallTaskParams) (Task, error) {
row := q.db.QueryRow(ctx, enqueueCallTask,
arg.ID,
arg.FallbackIssueNumber,
arg.Title,
arg.Body,
arg.Workflow,
arg.Inputs,
arg.CallKey,
arg.MaxDepth,
arg.FallbackIssueNumber,
)
var i Task
err := row.Scan(
@@ -114,6 +114,12 @@ type ChannelStatus struct {
ObservedAt time.Time
}
type ChatIssueNumber struct {
Owner string
Repo string
LastNumber int64
}
type ConfigSnapshot struct {
ID int64
Schema string
@@ -389,33 +389,32 @@ func (q *Queries) EnqueueIssue(ctx context.Context, arg EnqueueIssueParams) (int
}
const insertChatTask = `-- name: InsertChatTask :one
WITH number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number)
VALUES ($1, $2, $10::bigint + 1)
ON CONFLICT (owner, repo) DO UPDATE SET last_number = chat_issue_numbers.last_number + 1
RETURNING last_number
)
INSERT INTO tasks (owner, repo, issue_number, title, body, labels, workflow, source, identity, org_id, inputs, origin)
SELECT
$1, $2,
COALESCE((
SELECT MAX(existing.issue_number) FROM tasks existing
WHERE existing.owner = $1
AND existing.repo = $2
AND existing.source = 'chat'
), $3) + 1,
$4, $5, 'chat', $6, 'chat', $7,
$1, $2, number.last_number,
$3, $4, 'chat', $5, 'chat', $6,
owner_org.id,
$8,
$9
FROM (SELECT COALESCE(
(SELECT a.org_id FROM org_agents a WHERE a.identity_id = $7),
(SELECT m.org_id FROM memberships m WHERE m.identity_id = $7
$7,
$8
FROM number, (SELECT COALESCE(
(SELECT a.org_id FROM org_agents a WHERE a.identity_id = $6),
(SELECT m.org_id FROM memberships m WHERE m.identity_id = $6
ORDER BY m.created_at, m.org_id, m.workspace_id NULLS LAST LIMIT 1),
'org-sys'
) AS id) AS owner_org
WHERE $10::text IS NULL OR owner_org.id = $10
WHERE $9::text IS NULL OR owner_org.id = $9
RETURNING id, owner, repo, issue_number, title, body, labels, status, workflow, branch, plan, notes, pr_number, tokens_used, iterations, attempt, park_reason, watch_comment_id, park_class, remediation_rounds, retry_count, source, identity, binding_id, binding_version, review_payload, workflow_definition_version, workflow_definition_digest, workflow_definition_yaml, created_at, updated_at, review_cursor, inputs, org_id, workspace_id, call_parent_task_id, call_depth, outputs, review_gate, rereview_rounds, retry_mode, resume_from, resume_results, pending_reviews, origin, call_key, outcome_posted
`
type InsertChatTaskParams struct {
Owner string
Repo string
FallbackIssueNumber int64
Title string
Body string
Workflow string
@@ -423,10 +422,12 @@ type InsertChatTaskParams struct {
Inputs string
Origin string
ScopeOrg pgtype.Text
FallbackIssueNumber int64
}
// The synthetic issue number keeps chat-sourced tasks off the forge's real
// issue numbers; this allocator is the single source of truth for it.
// issue numbers; chat_issue_numbers is the single source of truth for it, and
// bumping its row serializes concurrent inserts in one repository.
// fallback_issue_number is only the seed for a repo's first chat task -- the
// passed value is not the issue number that lands.
// inputs is the chat task's own workflow inputs; a binding dispatch passes
@@ -438,7 +439,6 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams)
row := q.db.QueryRow(ctx, insertChatTask,
arg.Owner,
arg.Repo,
arg.FallbackIssueNumber,
arg.Title,
arg.Body,
arg.Workflow,
@@ -446,6 +446,7 @@ func (q *Queries) InsertChatTask(ctx context.Context, arg InsertChatTaskParams)
arg.Inputs,
arg.Origin,
arg.ScopeOrg,
arg.FallbackIssueNumber,
)
var i Task
err := row.Scan(
@@ -7,24 +7,25 @@
-- child that ended dead or closed_wont_do freed its key -- and the WHERE below
-- still decides whether the caller may call at all: a caller that is missing,
-- not running or past the depth limit inserts nothing and returns no row,
-- conflict or not.
-- conflict or not. The issue number comes from chat_issue_numbers, as for
-- InsertChatTask.
WITH caller AS (
SELECT t.* FROM tasks t WHERE t.id = $1 FOR UPDATE
), number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number)
SELECT caller.owner, caller.repo, sqlc.arg('fallback_issue_number')::bigint + 1 FROM caller
ON CONFLICT (owner, repo) DO UPDATE SET last_number = chat_issue_numbers.last_number + 1
RETURNING last_number
)
INSERT INTO tasks (owner, repo, issue_number, title, body, labels, workflow, source, identity, org_id, workspace_id, inputs, call_parent_task_id, call_depth, call_key)
SELECT
caller.owner,
caller.repo,
COALESCE((
SELECT MAX(existing.issue_number) FROM tasks existing
WHERE existing.owner = caller.owner
AND existing.repo = caller.repo
AND existing.source = 'chat'
), sqlc.arg('fallback_issue_number')::bigint) + 1,
number.last_number,
sqlc.arg(title), sqlc.arg(body), 'chat', sqlc.arg(workflow), 'chat', caller.identity,
caller.org_id, caller.workspace_id, sqlc.arg(inputs),
caller.id, caller.call_depth + 1, sqlc.arg('call_key')
FROM caller
FROM caller, number
WHERE caller.status = 'running' AND caller.call_depth + 1 <= sqlc.arg('max_depth')::int
ON CONFLICT (call_parent_task_id, call_key) WHERE call_key <> '' AND status NOT IN ('dead', 'closed_wont_do') DO UPDATE SET call_key = EXCLUDED.call_key
RETURNING *;
@@ -57,7 +57,8 @@ SELECT status, count(*)::int AS count FROM tasks WHERE (sqlc.narg(scope_org)::te
-- name: InsertChatTask :one
-- The synthetic issue number keeps chat-sourced tasks off the forge's real
-- issue numbers; this allocator is the single source of truth for it.
-- issue numbers; chat_issue_numbers is the single source of truth for it, and
-- bumping its row serializes concurrent inserts in one repository.
-- fallback_issue_number is only the seed for a repo's first chat task -- the
-- passed value is not the issue number that lands.
-- inputs is the chat task's own workflow inputs; a binding dispatch passes
@@ -65,20 +66,20 @@ SELECT status, count(*)::int AS count FROM tasks WHERE (sqlc.narg(scope_org)::te
-- StampTaskBinding adds only the provenance.
-- The task runs in its identity's org; a caller confined to another org
-- inserts nothing, so it cannot start work under an identity it does not own.
WITH number AS (
INSERT INTO chat_issue_numbers (owner, repo, last_number)
VALUES (sqlc.arg(owner), sqlc.arg(repo), sqlc.arg(fallback_issue_number)::bigint + 1)
ON CONFLICT (owner, repo) DO UPDATE SET last_number = chat_issue_numbers.last_number + 1
RETURNING last_number
)
INSERT INTO tasks (owner, repo, issue_number, title, body, labels, workflow, source, identity, org_id, inputs, origin)
SELECT
sqlc.arg(owner), sqlc.arg(repo),
COALESCE((
SELECT MAX(existing.issue_number) FROM tasks existing
WHERE existing.owner = sqlc.arg(owner)
AND existing.repo = sqlc.arg(repo)
AND existing.source = 'chat'
), sqlc.arg(fallback_issue_number)) + 1,
sqlc.arg(owner), sqlc.arg(repo), number.last_number,
sqlc.arg(title), sqlc.arg(body), 'chat', sqlc.arg(workflow), 'chat', sqlc.arg(identity),
owner_org.id,
sqlc.arg(inputs),
sqlc.arg(origin)
FROM (SELECT COALESCE(
FROM number, (SELECT COALESCE(
(SELECT a.org_id FROM org_agents a WHERE a.identity_id = sqlc.arg(identity)),
(SELECT m.org_id FROM memberships m WHERE m.identity_id = sqlc.arg(identity)
ORDER BY m.created_at, m.org_id, m.workspace_id NULLS LAST LIMIT 1),
+39 -9
View File
@@ -62,7 +62,7 @@ func guardTransition(ctx context.Context, q *postgresdb.Queries, taskID int64, f
func (s *Store) Close() error { return nil }
// syntheticIssueNumberBase is the seed for a repo's first chat task. The
// COALESCE allocator adds one to the passed fallback, so the fallback here is
// allocator adds one to the passed fallback, so the fallback here is
// base-1 and the first chat task lands on exactly base: the reserved
// JSON-safe floor that keeps synthetic issue numbers clear of real ones.
const syntheticIssueNumberBase = 1_000_000_000_000_000
@@ -156,11 +156,15 @@ func (s *Store) EnqueueIssue(ctx context.Context, owner, repo string, number int
// EnqueueChatTask inserts a queued chat task with a store-allocated synthetic
// issue number. origin names the conversation that created it, if any.
func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, wf, identity, origin string, inputs map[string]any) (*workflow.Task, error) {
return insertChatTask(ctx, s.queries(), owner, repo, title, body, wf, identity, origin, inputs)
}
func insertChatTask(ctx context.Context, q *postgresdb.Queries, owner, repo, title, body, wf, identity, origin string, inputs map[string]any) (*workflow.Task, error) {
encoded, err := task.EncodeInputs(inputs)
if err != nil {
return nil, err
}
t, err := s.queries().InsertChatTask(ctx, postgresdb.InsertChatTaskParams{
t, err := q.InsertChatTask(ctx, postgresdb.InsertChatTaskParams{
ScopeOrg: scopeOrg(ctx),
Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity,
Inputs: encoded,
@@ -176,20 +180,46 @@ func (s *Store) EnqueueChatTask(ctx context.Context, owner, repo, title, body, w
return taskFromRow(t), nil
}
// EnqueueBindingTask enqueues a binding-triggered task and stamps its binding
// provenance in a second statement (best-effort, as in the SQLite store). The
// inputs travel with the insert, so a failed stamp leaves a task whose inputs
// are already right and whose provenance is absent, never the reverse.
func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*workflow.Task, error) {
t, err := s.EnqueueChatTask(ctx, owner, repo, title, body, wf, identity, "", inputs)
// EnqueueBindingTask claims the dispatch ledger row for the capture's
// delivery, inserts the task, stamps its binding provenance and records the
// task on the ledger row in one transaction. A concurrent claim of the same
// delivery waits on the ledger's unique index and is refused once this one
// commits, so a replay creates no second task, and a failure anywhere leaves
// the delivery undispatched for the next cycle.
func (s *Store) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*workflow.Task, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
if err := s.queries().StampTaskBinding(ctx, postgresdb.StampTaskBindingParams{
defer func() { _ = tx.Rollback(ctx) }()
q := postgresdb.New(tx)
n, err := q.InsertBindingDispatch(ctx, postgresdb.InsertBindingDispatchParams{
Binding: bindingID, BindingVersion: int64(bindingVersion), Capture: captureID,
})
if err != nil {
return nil, ledgerWrite(err, "store: claim binding dispatch")
}
if n == 0 {
return nil, fmt.Errorf("store: claim binding dispatch: capture %q not found", captureID)
}
t, err := insertChatTask(ctx, q, owner, repo, title, body, wf, identity, "", inputs)
if err != nil {
return nil, err
}
if err := q.StampTaskBinding(ctx, postgresdb.StampTaskBindingParams{
ID: t.ID, BindingID: bindingID, BindingVersion: int64(bindingVersion),
}); err != nil {
return nil, fmt.Errorf("store: stamp binding provenance: %w", err)
}
if err := q.SetBindingDispatchTask(ctx, postgresdb.SetBindingDispatchTaskParams{
Binding: bindingID, Capture: captureID, TaskID: t.ID,
}); err != nil {
return nil, fmt.Errorf("store: record dispatch task: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
t.BindingID = bindingID
t.BindingVersion = bindingVersion
return t, nil
+2 -7
View File
@@ -649,11 +649,6 @@ func (c *Client) ListDispatches(ctx context.Context, filter storecontract.Dispat
}), nil
}
func (c *Client) SetDispatchTask(ctx context.Context, bindingID, captureID string, taskID int64) error {
_, err := c.client.SetDispatchTask(ctx, &pb.SetDispatchTaskRequest{BindingId: bindingID, CaptureId: captureID, TaskId: taskID})
return unmapError(err)
}
func (c *Client) UpdateBinding(ctx context.Context, b binding.Binding) error {
_, err := c.client.UpdateBinding(ctx, &pb.UpdateBindingRequest{Binding: bindingProto(b)})
return unmapError(err)
@@ -779,12 +774,12 @@ func (c *Client) ListUndispatchedCaptures(ctx context.Context, sources []string,
// BindingTaskCreator
func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, inputs map[string]any) (*task.Task, error) {
func (c *Client) EnqueueBindingTask(ctx context.Context, owner, repo, title, body, wf, identity, bindingID string, bindingVersion int, captureID string, inputs map[string]any) (*task.Task, error) {
encoded, err := task.EncodeInputs(inputs)
if err != nil {
return nil, err
}
r, err := c.client.EnqueueBindingTask(ctx, &pb.EnqueueBindingTaskRequest{Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity, BindingId: bindingID, BindingVersion: int64(bindingVersion), InputsJson: encoded})
r, err := c.client.EnqueueBindingTask(ctx, &pb.EnqueueBindingTaskRequest{Owner: owner, Repo: repo, Title: title, Body: body, Workflow: wf, Identity: identity, BindingId: bindingID, BindingVersion: int64(bindingVersion), InputsJson: encoded, CaptureId: captureID})
if err != nil {
return nil, unmapError(err)
}
+1 -11
View File
@@ -1114,16 +1114,6 @@ func (s *server) RecordDispatch(ctx context.Context, r *pb.RecordDispatchRequest
return &pb.RecordDispatchResponse{}, nil
}
func (s *server) SetDispatchTask(ctx context.Context, r *pb.SetDispatchTaskRequest) (*pb.SetDispatchTaskResponse, error) {
if s.deps.BindingDispatcher == nil {
return nil, errBindingDispatchUnavailable
}
if err := s.deps.BindingDispatcher.SetDispatchTask(ctx, r.BindingId, r.CaptureId, r.TaskId); err != nil {
return nil, s.logErr("SetDispatchTask", err)
}
return &pb.SetDispatchTaskResponse{}, nil
}
func (s *server) ListDispatches(ctx context.Context, r *pb.ListDispatchesRequest) (*pb.ListDispatchesResponse, error) {
bs, err := s.binding()
if err != nil {
@@ -1196,7 +1186,7 @@ func (s *server) EnqueueBindingTask(ctx context.Context, r *pb.EnqueueBindingTas
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
t, err := s.deps.BindingTaskCreator.EnqueueBindingTask(ctx, r.Owner, r.Repo, r.Title, r.Body, r.Workflow, r.Identity, r.BindingId, int(r.BindingVersion), inputs)
t, err := s.deps.BindingTaskCreator.EnqueueBindingTask(ctx, r.Owner, r.Repo, r.Title, r.Body, r.Workflow, r.Identity, r.BindingId, int(r.BindingVersion), r.CaptureId, inputs)
if err != nil {
return nil, s.logErr("EnqueueBindingTask", err)
}
+4 -9
View File
@@ -230,7 +230,6 @@ service StateStoreService {
// Dispatch
rpc ArmedBindingsForSource(ArmedBindingsForSourceRequest) returns (ArmedBindingsForSourceResponse);
rpc RecordDispatch(RecordDispatchRequest) returns (RecordDispatchResponse);
rpc SetDispatchTask(SetDispatchTaskRequest) returns (SetDispatchTaskResponse);
// Playbook dispatch ledger (eda-playbook-engine.md gap 2): one durable
// at-most-once record per (playbook, version, event, action) for the
// side-effecting positions. Administrative, like RecordDispatch: a
@@ -239,7 +238,9 @@ service StateStoreService {
rpc DeletePlaybookDispatches(DeletePlaybookDispatchesRequest) returns (DeletePlaybookDispatchesResponse);
rpc StreamUndispatchedCaptures(StreamUndispatchedCapturesRequest) returns (stream StreamUndispatchedCapturesResponse);
// BindingTaskCreator
// BindingTaskCreator: claims the binding's dispatch of a capture and
// enqueues its task in one transaction, so a failure leaves neither and a
// replayed delivery is refused with AlreadyExists.
rpc EnqueueBindingTask(EnqueueBindingTaskRequest) returns (EnqueueBindingTaskResponse);
// WorkflowCaller: a workflow.call step starts its callee and, with
@@ -1246,13 +1247,6 @@ message RecordDispatchRequest {
}
message RecordDispatchResponse {}
message SetDispatchTaskRequest {
string binding_id = 1;
string capture_id = 2;
int64 task_id = 3;
}
message SetDispatchTaskResponse {}
// ListDispatchesRequest filters the binding dispatch ledger; empty fields
// match any row.
message ListDispatchesRequest {
@@ -1299,6 +1293,7 @@ message EnqueueBindingTaskRequest {
string binding_id = 7;
int64 binding_version = 8;
string inputs_json = 9;
string capture_id = 10;
}
message EnqueueBindingTaskResponse { Task task = 1; }