fix(daemon): reconcile call waits from durable steps

This commit is contained in:
sam committed 2026-10-09 17:56:15 +11:00
1 parent 8c13485652
commit ffacc10766
3 files changed
+146 -113

No files matched your search

+35 -13
View File
@@ -2,8 +2,12 @@ package daemon
import ( import (
"context" "context"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
) )
// callWaitBuffer is the watcher's subscription buffer. The bus carries the // callWaitBuffer is the watcher's subscription buffer. The bus carries the
@@ -16,13 +20,16 @@ const callWaitBuffer = 1024
// daemon's own bus, republished from the container running the workflow, and // daemon's own bus, republished from the container running the workflow, and
// carry the caller's task id. // carry the caller's task id.
func (d *Daemon) watchCallWaits(ctx context.Context) { func (d *Daemon) watchCallWaits(ctx context.Context) {
if d.Bus == nil { reader, ok := d.Store.(storecontract.StepReader)
if d.Bus == nil || !ok {
return return
} }
sub := d.Bus.Subscribe(callWaitBuffer) sub := d.Bus.Subscribe(callWaitBuffer)
dispatcher := d.taskDispatcher() dispatcher := d.taskDispatcher()
go func() { go func() {
defer sub.Close() defer sub.Close()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@@ -31,23 +38,38 @@ func (d *Daemon) watchCallWaits(ctx context.Context) {
if !ok { if !ok {
return return
} }
applyCallWait(dispatcher, e) if e.Kind == events.KindWorkflowCallStarted || e.Kind == events.KindWorkflowCallFinished || e.Kind == events.KindStageStart || e.Kind == events.KindStageFinish {
d.reconcileCallWaits(ctx, dispatcher, reader)
}
case <-ticker.C:
d.reconcileCallWaits(ctx, dispatcher, reader)
} }
} }
}() }()
} }
// applyCallWait suspends the run a call start names and resumes it when the // Events only prompt a read; periodic reads recover dropped starts and finishes.
// call finishes. It stays non-blocking: a full subscriber buffer drops events, func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatcher, reader storecontract.StepReader) {
// and a dropped start leaves a waiting run holding the capacity its callee dispatcher.mu.Lock()
// needs, so this must never wait on anything. holds := make([]*taskHold, 0, len(dispatcher.holds))
func applyCallWait(dispatcher *taskDispatcher, e events.Event) { for _, hold := range dispatcher.holds {
switch e.Kind { holds = append(holds, hold)
case events.KindWorkflowCallStarted: }
if waiting, _ := e.Data["wait"].(bool); waiting { dispatcher.mu.Unlock()
dispatcher.Suspend(e.TaskID) for _, hold := range holds {
readCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
steps, err := reader.ListSteps(readCtx, hold.id, hold.attempt)
cancel()
if err != nil {
d.Log.Warn("read call waits", "task", hold.id, "err", err)
continue
} }
case events.KindWorkflowCallFinished: waits := 0
dispatcher.Resume(e.TaskID) for _, step := range steps {
if step.Kind == task.StepKindCall && step.CalledExecutionID != 0 && (step.Status == taskstate.StepPending || step.Status == taskstate.StepRunning) {
waits++
}
}
dispatcher.setCallWaits(hold, waits)
} }
} }
+19 -39
View File
@@ -1026,25 +1026,27 @@ type taskHold struct {
baton chan struct{} baton chan struct{}
// slot is true while the run counts against the global limit. // slot is true while the run counts against the global limit.
slot bool slot bool
// waits counts the callees the run is waiting on: the first Suspend gives // waits counts the callees the run is waiting on: an open call gives capacity
// capacity back and the last Resume takes it again. // back and the last closed call takes it again.
waits int waits int
// finished is true once the run's process returned. // finished is true once the run's process returned.
finished bool finished bool
// id is the task this hold belongs to, named by the run once it knows the // id is the task this hold belongs to, named by the run once it knows the
// task's id. // task's id.
id int64 id int64
attempt int
} }
// name records which task's capacity this hold is, so a wait on a callee can // name records which task's capacity this hold is, so a wait on a callee can
// give it back by task id. Zero names nothing: a task arriving on the NATS // give it back by task id. Zero names nothing: a task arriving on the NATS
// stream has no id until the store claims its row inside the run. // stream has no id until the store claims its row inside the run.
func (h *taskHold) name(id int64) { func (h *taskHold) name(id int64, attempt int) {
if id == 0 { if id == 0 {
return return
} }
h.dispatcher.mu.Lock() h.dispatcher.mu.Lock()
h.id = id h.id = id
h.attempt = attempt
h.dispatcher.holds[id] = h h.dispatcher.holds[id] = h
h.dispatcher.mu.Unlock() h.dispatcher.mu.Unlock()
} }
@@ -1125,47 +1127,25 @@ func (d *taskDispatcher) finishRun(hold *taskHold) {
d.slotsFree.Broadcast() d.slotsFree.Broadcast()
} }
// Suspend gives a run's capacity back while it waits on a callee, so the // setCallWaits applies a durable snapshot without accumulating duplicate events.
// callee can start. Calls are refcounted per task: a run waiting on callees func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
// from parallel branches releases once. An unknown task, or one whose run has
// already returned, is a no-op.
func (d *taskDispatcher) Suspend(id int64) {
d.mu.Lock() d.mu.Lock()
hold := d.holds[id] if d.holds[hold.id] != hold || hold.finished {
if hold == nil || hold.finished {
d.mu.Unlock() d.mu.Unlock()
return return
} }
hold.waits++ previous := hold.waits
if hold.waits == 1 { hold.waits = waits
if waits > 0 {
d.giveBackLocked(hold) d.giveBackLocked(hold)
} }
d.mu.Unlock() d.mu.Unlock()
d.slotsFree.Broadcast() if waits > 0 {
} d.slotsFree.Broadcast()
} else if previous > 0 {
// Resume takes a run's capacity back when one of its waits ends. Only the last // Reacquiring may block behind another task; the caller keeps running.
// wait does: a run waiting on callees from parallel branches keeps nothing go d.reacquire(hold)
// until all of them are done. An unknown task, one whose run has returned, and
// one that never suspended are all no-ops.
func (d *taskDispatcher) Resume(id int64) {
d.mu.Lock()
hold := d.holds[id]
if hold == nil || hold.finished || hold.waits == 0 {
d.mu.Unlock()
return
} }
hold.waits--
if hold.waits > 0 {
d.mu.Unlock()
return
}
d.mu.Unlock()
// The two waits below can block, and neither may hold up the bus
// subscriber calling this or the resumed run, which is already executing
// inside its container: the execution cap can be exceeded by at most the
// number of simultaneous resumes, and re-converges as tasks finish.
go d.reacquire(hold)
} }
// reacquire returns a resumed run's capacity: its place in its repo's chain // reacquire returns a resumed run's capacity: its place in its repo's chain
@@ -1176,7 +1156,7 @@ func (d *taskDispatcher) Resume(id int64) {
func (d *taskDispatcher) reacquire(hold *taskHold) { func (d *taskDispatcher) reacquire(hold *taskHold) {
d.mu.Lock() d.mu.Lock()
if hold.finished || hold.waits > 0 { if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the Resume that // The run returned, or started waiting again, after the reconciliation that
// queued this: it needs no capacity. // queued this: it needs no capacity.
d.mu.Unlock() d.mu.Unlock()
return return
@@ -1593,7 +1573,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) { func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
// A task off the NATS stream only has an id once the store has claimed its // A task off the NATS stream only has an id once the store has claimed its
// row, inside this call, so its capacity is named here. // row, inside this call, so its capacity is named here.
hold.name(task.ID) hold.name(task.ID, task.Attempt)
// The workflow may name the identity the run acts as, which everything // The workflow may name the identity the run acts as, which everything
// below is chosen by, so it is settled before anything else. // below is chosen by, so it is settled before anything else.
if !d.adoptWorkflowIdentity(ctx, task) { if !d.adoptWorkflowIdentity(ctx, task) {
+92 -61
View File
@@ -2,7 +2,6 @@ package daemon
import ( import (
"context" "context"
"encoding/json"
"log/slog" "log/slog"
"runtime" "runtime"
"sync" "sync"
@@ -12,8 +11,10 @@ import (
"github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow" "github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/eventbus" "github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events" "github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
) )
// runDeadline bounds a wait for something that must happen. Only a broken // runDeadline bounds a wait for something that must happen. Only a broken
@@ -139,12 +140,25 @@ func slotHolders(d *taskDispatcher) int {
return held return held
} }
func changeCallWaits(d *taskDispatcher, delta int, id int64) {
d.mu.Lock()
hold := d.holds[id]
waits := 0
if hold != nil {
waits = max(0, hold.waits+delta)
}
d.mu.Unlock()
if hold != nil {
d.setCallWaits(hold, waits)
}
}
// submitRun queues a task whose process reports its start on started and then // submitRun queues a task whose process reports its start on started and then
// blocks until released. Naming the run's capacity is what the daemon's own // blocks until released. Naming the run's capacity is what the daemon's own
// process does, where the task's id first exists. // process does, where the task's id first exists.
func submitRun(d *taskDispatcher, id int64, repo string, started chan<- int64, released <-chan struct{}) { func submitRun(d *taskDispatcher, id int64, repo string, started chan<- int64, released <-chan struct{}) {
d.Submit(context.Background(), &workflow.Task{ID: id, Owner: "acme", Repo: repo}, func(_ context.Context, task *workflow.Task, hold *taskHold) { d.Submit(context.Background(), &workflow.Task{ID: id, Owner: "acme", Repo: repo}, func(_ context.Context, task *workflow.Task, hold *taskHold) {
hold.name(task.ID) hold.name(task.ID, task.Attempt)
started <- task.ID started <- task.ID
<-released <-released
}) })
@@ -164,11 +178,11 @@ func TestSuspendReleasesCapacityForTheTaskBehindIt(t *testing.T) {
submitRun(d, 2, "app", started, b) submitRun(d, 2, "app", started, b)
awaitNoStart(t, started, "while the waiting run held the only slot and the repo's place") awaitNoStart(t, started, "while the waiting run held the only slot and the repo's place")
d.Suspend(1) changeCallWaits(d, 1, 1)
awaitStart(t, started, 2) awaitStart(t, started, 2)
close(b) close(b)
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) }) awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
submitRun(d, 3, "app", started, c) submitRun(d, 3, "app", started, c)
@@ -193,11 +207,11 @@ func TestResumedRunKeepsItsPlaceInTheReposChain(t *testing.T) {
submitRun(d, 2, "app", started, b) submitRun(d, 2, "app", started, b)
awaitNoStart(t, started, "while the repo's place was another run's") awaitNoStart(t, started, "while the repo's place was another run's")
d.Suspend(1) changeCallWaits(d, 1, 1)
awaitStart(t, started, 2) awaitStart(t, started, 2)
close(b) close(b)
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) }) awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
submitRun(d, 3, "app", started, c) submitRun(d, 3, "app", started, c)
awaitNoStart(t, started, "while the resumed run held the repo's place, with slots free") awaitNoStart(t, started, "while the resumed run held the repo's place, with slots free")
@@ -217,8 +231,8 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
submitRun(d, 1, "app", started, a) submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1) awaitStart(t, started, 1)
d.Suspend(1) changeCallWaits(d, 1, 1)
d.Suspend(1) changeCallWaits(d, 1, 1)
// The callee queued behind it runs, and returns. // The callee queued behind it runs, and returns.
submitRun(d, 2, "app", started, b) submitRun(d, 2, "app", started, b)
@@ -228,11 +242,11 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
// One wait outstanding: the run keeps no capacity while its other callee // One wait outstanding: the run keeps no capacity while its other callee
// runs. // runs.
d.Resume(1) changeCallWaits(d, -1, 1)
awaitNoState(t, "task 1 took capacity back with a callee still running", func() bool { return holdsCapacity(d, 1) }) awaitNoState(t, "task 1 took capacity back with a callee still running", func() bool { return holdsCapacity(d, 1) })
// With the last wait done it takes capacity back. // With the last wait done it takes capacity back.
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) }) awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
close(a) close(a)
@@ -253,13 +267,13 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
submitRun(d, 1, "app", started, a) submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1) awaitStart(t, started, 1)
d.Suspend(1) changeCallWaits(d, 1, 1)
// X takes the repo's place while A waits on its callee, so A's first // X takes the repo's place while A waits on its callee, so A's first
// re-acquire queues its next place behind X. // re-acquire queues its next place behind X.
submitRun(d, 2, "app", started, x) submitRun(d, 2, "app", started, x)
awaitStart(t, started, 2) awaitStart(t, started, 2)
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not queue its place back", func() bool { awaitState(t, "task 1 did not queue its place back", func() bool {
_, place, _ := capacity(d, 1) _, place, _ := capacity(d, 1)
return place return place
@@ -268,8 +282,8 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
// That callee ends and the run starts its next call at once: the place the // That callee ends and the run starts its next call at once: the place the
// first re-acquire claimed is given back with it, and the second re-acquire // first re-acquire claimed is given back with it, and the second re-acquire
// takes a slot immediately. // takes a slot immediately.
d.Suspend(1) changeCallWaits(d, 1, 1)
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) }) awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) })
// X leaves, so the first re-acquire can take a slot as well. // X leaves, so the first re-acquire can take a slot as well.
@@ -308,13 +322,13 @@ func TestSecondReacquireDoesNotQueueASecondPlace(t *testing.T) {
submitRun(d, 1, "app", started, a) submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1) awaitStart(t, started, 1)
d.Suspend(1) changeCallWaits(d, 1, 1)
// X holds the repo's place, so the run's re-acquire queues behind it and // X holds the repo's place, so the run's re-acquire queues behind it and
// stays there. // stays there.
submitRun(d, 2, "app", started, x) submitRun(d, 2, "app", started, x)
awaitStart(t, started, 2) awaitStart(t, started, 2)
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not queue its place back", func() bool { awaitState(t, "task 1 did not queue its place back", func() bool {
_, place, _ := capacity(d, 1) _, place, _ := capacity(d, 1)
return place return place
@@ -363,14 +377,14 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
submitRun(d, 1, "app", started, a) submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1) awaitStart(t, started, 1)
d.Suspend(1) changeCallWaits(d, 1, 1)
submitRun(d, 2, "app", started, b) submitRun(d, 2, "app", started, b)
awaitStart(t, started, 2) awaitStart(t, started, 2)
// The resume is queued for a slot and a place in the chain, and overtaken by // The resume is queued for a slot and a place in the chain, and overtaken by
// the run returning before either comes free. // the run returning before either comes free.
d.Resume(1) changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its place in the chain back", func() bool { awaitState(t, "task 1 did not take its place in the chain back", func() bool {
_, place, _ := capacity(d, 1) _, place, _ := capacity(d, 1)
return place return place
@@ -394,49 +408,7 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
close(c) close(c)
// A Resume arriving after the run returned is a no-op. // A Resume arriving after the run returned is a no-op.
d.Resume(1) changeCallWaits(d, -1, 1)
d.Wait()
}
// callEvent builds one call step's event as the daemon receives it: the event
// the container's workflow engine published, JSON-encoded and decoded again by
// the agent-event bridge, so the wait flag arrives as a decoded bool.
func callEvent(t *testing.T, kind string, taskID int64, wait bool) events.Event {
t.Helper()
raw, err := json.Marshal(events.Event{Kind: kind, TaskID: taskID, Data: map[string]any{"wait": wait}})
if err != nil {
t.Fatal(err)
}
var e events.Event
if err := json.Unmarshal(raw, &e); err != nil {
t.Fatal(err)
}
return e
}
// Only a call the caller waits on releases its capacity, and the call
// finishing takes it back.
func TestCallWaitEventsReleaseCapacity(t *testing.T) {
d := newTaskDispatcher(1, nil)
started := make(chan int64, 4)
a, b := make(chan struct{}), make(chan struct{})
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
submitRun(d, 2, "app", started, b)
// A call that does not wait is not waited on: nothing is released.
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, false))
awaitNoStart(t, started, "while the call that started does not wait")
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, true))
awaitStart(t, started, 2)
close(b)
applyCallWait(d, callEvent(t, events.KindWorkflowCallFinished, 1, false))
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
close(a)
d.Wait() d.Wait()
} }
@@ -447,6 +419,65 @@ func TestCallWaitsWithoutABus(t *testing.T) {
d.WaitForTasks() d.WaitForTasks()
} }
type callWaitStore struct {
storecontract.TaskStore
mu sync.Mutex
steps []task.StepExecution
}
func (s *callWaitStore) ListSteps(_ context.Context, id int64, _ int) ([]task.StepExecution, error) {
if id != 1 {
return nil, nil
}
s.mu.Lock()
defer s.mu.Unlock()
return append([]task.StepExecution(nil), s.steps...), nil
}
func TestCallWaitsRecoverDroppedStart(t *testing.T) {
store := &callWaitStore{steps: []task.StepExecution{
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 2},
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 3},
}}
cfg := config.Config{}
cfg.Containers.MaxConcurrency = 1
d := &Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}
dispatcher := d.taskDispatcher()
started := make(chan int64, 2)
a, b := make(chan struct{}), make(chan struct{})
defer func() { close(a); dispatcher.Wait() }()
defer func() {
if b != nil {
close(b)
}
}()
submitRun(dispatcher, 1, "app", started, a)
awaitStart(t, started, 1)
submitRun(dispatcher, 2, "app", started, b)
// The live start event was dropped; only the recorded call remains.
d.watchCallWaits(t.Context())
awaitStart(t, started, 2)
close(b)
b = nil
// Repeated reads must not accumulate waits; a partial finish stays suspended.
store.mu.Lock()
store.steps[0].Status = taskstate.StepSucceeded
store.mu.Unlock()
awaitState(t, "remaining call was not reconciled", func() bool {
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.holds[1].waits == 1
})
if holdsCapacity(dispatcher, 1) {
t.Fatal("parent reacquired capacity with a call still open")
}
// The finish event is also missing; the durable close restores capacity.
store.mu.Lock()
store.steps[1].Status = taskstate.StepSucceeded
store.mu.Unlock()
awaitState(t, "parent did not recover its capacity", func() bool { return holdsCapacity(dispatcher, 1) })
}
// stubTasks is a task bus with an empty stream, so a drain pass falls through // stubTasks is a task bus with an empty stream, so a drain pass falls through
// to the store's ClaimNext. // to the store's ClaimNext.
type stubTasks struct{} type stubTasks struct{}