fix(daemon): reconcile call waits from durable steps

This commit is contained in:
sam committed 2026-10-09 17:56:15 +11:00
1 parent 8c13485652
commit ffacc10766
3 files changed
+146 -113

No files matched your search

+35 -13
View File
@@ -2,8 +2,12 @@ package daemon
import (
"context"
"time"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// callWaitBuffer is the watcher's subscription buffer. The bus carries the
@@ -16,13 +20,16 @@ const callWaitBuffer = 1024
// daemon's own bus, republished from the container running the workflow, and
// carry the caller's task id.
func (d *Daemon) watchCallWaits(ctx context.Context) {
if d.Bus == nil {
reader, ok := d.Store.(storecontract.StepReader)
if d.Bus == nil || !ok {
return
}
sub := d.Bus.Subscribe(callWaitBuffer)
dispatcher := d.taskDispatcher()
go func() {
defer sub.Close()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
@@ -31,23 +38,38 @@ func (d *Daemon) watchCallWaits(ctx context.Context) {
if !ok {
return
}
applyCallWait(dispatcher, e)
if e.Kind == events.KindWorkflowCallStarted || e.Kind == events.KindWorkflowCallFinished || e.Kind == events.KindStageStart || e.Kind == events.KindStageFinish {
d.reconcileCallWaits(ctx, dispatcher, reader)
}
case <-ticker.C:
d.reconcileCallWaits(ctx, dispatcher, reader)
}
}
}()
}
// applyCallWait suspends the run a call start names and resumes it when the
// call finishes. It stays non-blocking: a full subscriber buffer drops events,
// and a dropped start leaves a waiting run holding the capacity its callee
// needs, so this must never wait on anything.
func applyCallWait(dispatcher *taskDispatcher, e events.Event) {
switch e.Kind {
case events.KindWorkflowCallStarted:
if waiting, _ := e.Data["wait"].(bool); waiting {
dispatcher.Suspend(e.TaskID)
// Events only prompt a read; periodic reads recover dropped starts and finishes.
func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatcher, reader storecontract.StepReader) {
dispatcher.mu.Lock()
holds := make([]*taskHold, 0, len(dispatcher.holds))
for _, hold := range dispatcher.holds {
holds = append(holds, hold)
}
dispatcher.mu.Unlock()
for _, hold := range holds {
readCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
steps, err := reader.ListSteps(readCtx, hold.id, hold.attempt)
cancel()
if err != nil {
d.Log.Warn("read call waits", "task", hold.id, "err", err)
continue
}
case events.KindWorkflowCallFinished:
dispatcher.Resume(e.TaskID)
waits := 0
for _, step := range steps {
if step.Kind == task.StepKindCall && step.CalledExecutionID != 0 && (step.Status == taskstate.StepPending || step.Status == taskstate.StepRunning) {
waits++
}
}
dispatcher.setCallWaits(hold, waits)
}
}
+19 -39
View File
@@ -1026,25 +1026,27 @@ type taskHold struct {
baton chan struct{}
// slot is true while the run counts against the global limit.
slot bool
// waits counts the callees the run is waiting on: the first Suspend gives
// capacity back and the last Resume takes it again.
// waits counts the callees the run is waiting on: an open call gives capacity
// back and the last closed call takes it again.
waits int
// finished is true once the run's process returned.
finished bool
// id is the task this hold belongs to, named by the run once it knows the
// task's id.
id int64
id int64
attempt int
}
// name records which task's capacity this hold is, so a wait on a callee can
// give it back by task id. Zero names nothing: a task arriving on the NATS
// stream has no id until the store claims its row inside the run.
func (h *taskHold) name(id int64) {
func (h *taskHold) name(id int64, attempt int) {
if id == 0 {
return
}
h.dispatcher.mu.Lock()
h.id = id
h.attempt = attempt
h.dispatcher.holds[id] = h
h.dispatcher.mu.Unlock()
}
@@ -1125,47 +1127,25 @@ func (d *taskDispatcher) finishRun(hold *taskHold) {
d.slotsFree.Broadcast()
}
// Suspend gives a run's capacity back while it waits on a callee, so the
// callee can start. Calls are refcounted per task: a run waiting on callees
// from parallel branches releases once. An unknown task, or one whose run has
// already returned, is a no-op.
func (d *taskDispatcher) Suspend(id int64) {
// setCallWaits applies a durable snapshot without accumulating duplicate events.
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
d.mu.Lock()
hold := d.holds[id]
if hold == nil || hold.finished {
if d.holds[hold.id] != hold || hold.finished {
d.mu.Unlock()
return
}
hold.waits++
if hold.waits == 1 {
previous := hold.waits
hold.waits = waits
if waits > 0 {
d.giveBackLocked(hold)
}
d.mu.Unlock()
d.slotsFree.Broadcast()
}
// Resume takes a run's capacity back when one of its waits ends. Only the last
// wait does: a run waiting on callees from parallel branches keeps nothing
// until all of them are done. An unknown task, one whose run has returned, and
// one that never suspended are all no-ops.
func (d *taskDispatcher) Resume(id int64) {
d.mu.Lock()
hold := d.holds[id]
if hold == nil || hold.finished || hold.waits == 0 {
d.mu.Unlock()
return
if waits > 0 {
d.slotsFree.Broadcast()
} else if previous > 0 {
// Reacquiring may block behind another task; the caller keeps running.
go d.reacquire(hold)
}
hold.waits--
if hold.waits > 0 {
d.mu.Unlock()
return
}
d.mu.Unlock()
// The two waits below can block, and neither may hold up the bus
// subscriber calling this or the resumed run, which is already executing
// inside its container: the execution cap can be exceeded by at most the
// number of simultaneous resumes, and re-converges as tasks finish.
go d.reacquire(hold)
}
// reacquire returns a resumed run's capacity: its place in its repo's chain
@@ -1176,7 +1156,7 @@ func (d *taskDispatcher) Resume(id int64) {
func (d *taskDispatcher) reacquire(hold *taskHold) {
d.mu.Lock()
if hold.finished || hold.waits > 0 {
// The run returned, or started waiting again, after the Resume that
// The run returned, or started waiting again, after the reconciliation that
// queued this: it needs no capacity.
d.mu.Unlock()
return
@@ -1593,7 +1573,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
// A task off the NATS stream only has an id once the store has claimed its
// row, inside this call, so its capacity is named here.
hold.name(task.ID)
hold.name(task.ID, task.Attempt)
// The workflow may name the identity the run acts as, which everything
// below is chosen by, so it is settled before anything else.
if !d.adoptWorkflowIdentity(ctx, task) {
+92 -61
View File
@@ -2,7 +2,6 @@ package daemon
import (
"context"
"encoding/json"
"log/slog"
"runtime"
"sync"
@@ -12,8 +11,10 @@ import (
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/eventbus"
"github.com/samcharles93/archie-core/internal/events"
"github.com/samcharles93/archie-core/internal/taskstate"
)
// runDeadline bounds a wait for something that must happen. Only a broken
@@ -139,12 +140,25 @@ func slotHolders(d *taskDispatcher) int {
return held
}
func changeCallWaits(d *taskDispatcher, delta int, id int64) {
d.mu.Lock()
hold := d.holds[id]
waits := 0
if hold != nil {
waits = max(0, hold.waits+delta)
}
d.mu.Unlock()
if hold != nil {
d.setCallWaits(hold, waits)
}
}
// submitRun queues a task whose process reports its start on started and then
// blocks until released. Naming the run's capacity is what the daemon's own
// process does, where the task's id first exists.
func submitRun(d *taskDispatcher, id int64, repo string, started chan<- int64, released <-chan struct{}) {
d.Submit(context.Background(), &workflow.Task{ID: id, Owner: "acme", Repo: repo}, func(_ context.Context, task *workflow.Task, hold *taskHold) {
hold.name(task.ID)
hold.name(task.ID, task.Attempt)
started <- task.ID
<-released
})
@@ -164,11 +178,11 @@ func TestSuspendReleasesCapacityForTheTaskBehindIt(t *testing.T) {
submitRun(d, 2, "app", started, b)
awaitNoStart(t, started, "while the waiting run held the only slot and the repo's place")
d.Suspend(1)
changeCallWaits(d, 1, 1)
awaitStart(t, started, 2)
close(b)
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
submitRun(d, 3, "app", started, c)
@@ -193,11 +207,11 @@ func TestResumedRunKeepsItsPlaceInTheReposChain(t *testing.T) {
submitRun(d, 2, "app", started, b)
awaitNoStart(t, started, "while the repo's place was another run's")
d.Suspend(1)
changeCallWaits(d, 1, 1)
awaitStart(t, started, 2)
close(b)
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
submitRun(d, 3, "app", started, c)
awaitNoStart(t, started, "while the resumed run held the repo's place, with slots free")
@@ -217,8 +231,8 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
d.Suspend(1)
d.Suspend(1)
changeCallWaits(d, 1, 1)
changeCallWaits(d, 1, 1)
// The callee queued behind it runs, and returns.
submitRun(d, 2, "app", started, b)
@@ -228,11 +242,11 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
// One wait outstanding: the run keeps no capacity while its other callee
// runs.
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitNoState(t, "task 1 took capacity back with a callee still running", func() bool { return holdsCapacity(d, 1) })
// With the last wait done it takes capacity back.
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
close(a)
@@ -253,13 +267,13 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
d.Suspend(1)
changeCallWaits(d, 1, 1)
// X takes the repo's place while A waits on its callee, so A's first
// re-acquire queues its next place behind X.
submitRun(d, 2, "app", started, x)
awaitStart(t, started, 2)
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not queue its place back", func() bool {
_, place, _ := capacity(d, 1)
return place
@@ -268,8 +282,8 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
// That callee ends and the run starts its next call at once: the place the
// first re-acquire claimed is given back with it, and the second re-acquire
// takes a slot immediately.
d.Suspend(1)
d.Resume(1)
changeCallWaits(d, 1, 1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) })
// X leaves, so the first re-acquire can take a slot as well.
@@ -308,13 +322,13 @@ func TestSecondReacquireDoesNotQueueASecondPlace(t *testing.T) {
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
d.Suspend(1)
changeCallWaits(d, 1, 1)
// X holds the repo's place, so the run's re-acquire queues behind it and
// stays there.
submitRun(d, 2, "app", started, x)
awaitStart(t, started, 2)
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not queue its place back", func() bool {
_, place, _ := capacity(d, 1)
return place
@@ -363,14 +377,14 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
d.Suspend(1)
changeCallWaits(d, 1, 1)
submitRun(d, 2, "app", started, b)
awaitStart(t, started, 2)
// The resume is queued for a slot and a place in the chain, and overtaken by
// the run returning before either comes free.
d.Resume(1)
changeCallWaits(d, -1, 1)
awaitState(t, "task 1 did not take its place in the chain back", func() bool {
_, place, _ := capacity(d, 1)
return place
@@ -394,49 +408,7 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
close(c)
// A Resume arriving after the run returned is a no-op.
d.Resume(1)
d.Wait()
}
// callEvent builds one call step's event as the daemon receives it: the event
// the container's workflow engine published, JSON-encoded and decoded again by
// the agent-event bridge, so the wait flag arrives as a decoded bool.
func callEvent(t *testing.T, kind string, taskID int64, wait bool) events.Event {
t.Helper()
raw, err := json.Marshal(events.Event{Kind: kind, TaskID: taskID, Data: map[string]any{"wait": wait}})
if err != nil {
t.Fatal(err)
}
var e events.Event
if err := json.Unmarshal(raw, &e); err != nil {
t.Fatal(err)
}
return e
}
// Only a call the caller waits on releases its capacity, and the call
// finishing takes it back.
func TestCallWaitEventsReleaseCapacity(t *testing.T) {
d := newTaskDispatcher(1, nil)
started := make(chan int64, 4)
a, b := make(chan struct{}), make(chan struct{})
submitRun(d, 1, "app", started, a)
awaitStart(t, started, 1)
submitRun(d, 2, "app", started, b)
// A call that does not wait is not waited on: nothing is released.
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, false))
awaitNoStart(t, started, "while the call that started does not wait")
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, true))
awaitStart(t, started, 2)
close(b)
applyCallWait(d, callEvent(t, events.KindWorkflowCallFinished, 1, false))
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
close(a)
changeCallWaits(d, -1, 1)
d.Wait()
}
@@ -447,6 +419,65 @@ func TestCallWaitsWithoutABus(t *testing.T) {
d.WaitForTasks()
}
type callWaitStore struct {
storecontract.TaskStore
mu sync.Mutex
steps []task.StepExecution
}
func (s *callWaitStore) ListSteps(_ context.Context, id int64, _ int) ([]task.StepExecution, error) {
if id != 1 {
return nil, nil
}
s.mu.Lock()
defer s.mu.Unlock()
return append([]task.StepExecution(nil), s.steps...), nil
}
func TestCallWaitsRecoverDroppedStart(t *testing.T) {
store := &callWaitStore{steps: []task.StepExecution{
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 2},
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 3},
}}
cfg := config.Config{}
cfg.Containers.MaxConcurrency = 1
d := &Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}
dispatcher := d.taskDispatcher()
started := make(chan int64, 2)
a, b := make(chan struct{}), make(chan struct{})
defer func() { close(a); dispatcher.Wait() }()
defer func() {
if b != nil {
close(b)
}
}()
submitRun(dispatcher, 1, "app", started, a)
awaitStart(t, started, 1)
submitRun(dispatcher, 2, "app", started, b)
// The live start event was dropped; only the recorded call remains.
d.watchCallWaits(t.Context())
awaitStart(t, started, 2)
close(b)
b = nil
// Repeated reads must not accumulate waits; a partial finish stays suspended.
store.mu.Lock()
store.steps[0].Status = taskstate.StepSucceeded
store.mu.Unlock()
awaitState(t, "remaining call was not reconciled", func() bool {
dispatcher.mu.Lock()
defer dispatcher.mu.Unlock()
return dispatcher.holds[1].waits == 1
})
if holdsCapacity(dispatcher, 1) {
t.Fatal("parent reacquired capacity with a call still open")
}
// The finish event is also missing; the durable close restores capacity.
store.mu.Lock()
store.steps[1].Status = taskstate.StepSucceeded
store.mu.Unlock()
awaitState(t, "parent did not recover its capacity", func() bool { return holdsCapacity(dispatcher, 1) })
}
// stubTasks is a task bus with an empty stream, so a drain pass falls through
// to the store's ClaimNext.
type stubTasks struct{}