fix(daemon): reconcile call waits from durable steps
This commit is contained in:
3 files changed
+146
-113
No files matched your search
@@ -2,8 +2,12 @@ package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// callWaitBuffer is the watcher's subscription buffer. The bus carries the
|
||||
@@ -16,13 +20,16 @@ const callWaitBuffer = 1024
|
||||
// daemon's own bus, republished from the container running the workflow, and
|
||||
// carry the caller's task id.
|
||||
func (d *Daemon) watchCallWaits(ctx context.Context) {
|
||||
if d.Bus == nil {
|
||||
reader, ok := d.Store.(storecontract.StepReader)
|
||||
if d.Bus == nil || !ok {
|
||||
return
|
||||
}
|
||||
sub := d.Bus.Subscribe(callWaitBuffer)
|
||||
dispatcher := d.taskDispatcher()
|
||||
go func() {
|
||||
defer sub.Close()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -31,23 +38,38 @@ func (d *Daemon) watchCallWaits(ctx context.Context) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
applyCallWait(dispatcher, e)
|
||||
if e.Kind == events.KindWorkflowCallStarted || e.Kind == events.KindWorkflowCallFinished || e.Kind == events.KindStageStart || e.Kind == events.KindStageFinish {
|
||||
d.reconcileCallWaits(ctx, dispatcher, reader)
|
||||
}
|
||||
case <-ticker.C:
|
||||
d.reconcileCallWaits(ctx, dispatcher, reader)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// applyCallWait suspends the run a call start names and resumes it when the
|
||||
// call finishes. It stays non-blocking: a full subscriber buffer drops events,
|
||||
// and a dropped start leaves a waiting run holding the capacity its callee
|
||||
// needs, so this must never wait on anything.
|
||||
func applyCallWait(dispatcher *taskDispatcher, e events.Event) {
|
||||
switch e.Kind {
|
||||
case events.KindWorkflowCallStarted:
|
||||
if waiting, _ := e.Data["wait"].(bool); waiting {
|
||||
dispatcher.Suspend(e.TaskID)
|
||||
// Events only prompt a read; periodic reads recover dropped starts and finishes.
|
||||
func (d *Daemon) reconcileCallWaits(ctx context.Context, dispatcher *taskDispatcher, reader storecontract.StepReader) {
|
||||
dispatcher.mu.Lock()
|
||||
holds := make([]*taskHold, 0, len(dispatcher.holds))
|
||||
for _, hold := range dispatcher.holds {
|
||||
holds = append(holds, hold)
|
||||
}
|
||||
dispatcher.mu.Unlock()
|
||||
for _, hold := range holds {
|
||||
readCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
steps, err := reader.ListSteps(readCtx, hold.id, hold.attempt)
|
||||
cancel()
|
||||
if err != nil {
|
||||
d.Log.Warn("read call waits", "task", hold.id, "err", err)
|
||||
continue
|
||||
}
|
||||
case events.KindWorkflowCallFinished:
|
||||
dispatcher.Resume(e.TaskID)
|
||||
waits := 0
|
||||
for _, step := range steps {
|
||||
if step.Kind == task.StepKindCall && step.CalledExecutionID != 0 && (step.Status == taskstate.StepPending || step.Status == taskstate.StepRunning) {
|
||||
waits++
|
||||
}
|
||||
}
|
||||
dispatcher.setCallWaits(hold, waits)
|
||||
}
|
||||
}
|
||||
+19
-39
@@ -1026,25 +1026,27 @@ type taskHold struct {
|
||||
baton chan struct{}
|
||||
// slot is true while the run counts against the global limit.
|
||||
slot bool
|
||||
// waits counts the callees the run is waiting on: the first Suspend gives
|
||||
// capacity back and the last Resume takes it again.
|
||||
// waits counts the callees the run is waiting on: an open call gives capacity
|
||||
// back and the last closed call takes it again.
|
||||
waits int
|
||||
// finished is true once the run's process returned.
|
||||
finished bool
|
||||
// id is the task this hold belongs to, named by the run once it knows the
|
||||
// task's id.
|
||||
id int64
|
||||
id int64
|
||||
attempt int
|
||||
}
|
||||
|
||||
// name records which task's capacity this hold is, so a wait on a callee can
|
||||
// give it back by task id. Zero names nothing: a task arriving on the NATS
|
||||
// stream has no id until the store claims its row inside the run.
|
||||
func (h *taskHold) name(id int64) {
|
||||
func (h *taskHold) name(id int64, attempt int) {
|
||||
if id == 0 {
|
||||
return
|
||||
}
|
||||
h.dispatcher.mu.Lock()
|
||||
h.id = id
|
||||
h.attempt = attempt
|
||||
h.dispatcher.holds[id] = h
|
||||
h.dispatcher.mu.Unlock()
|
||||
}
|
||||
@@ -1125,47 +1127,25 @@ func (d *taskDispatcher) finishRun(hold *taskHold) {
|
||||
d.slotsFree.Broadcast()
|
||||
}
|
||||
|
||||
// Suspend gives a run's capacity back while it waits on a callee, so the
|
||||
// callee can start. Calls are refcounted per task: a run waiting on callees
|
||||
// from parallel branches releases once. An unknown task, or one whose run has
|
||||
// already returned, is a no-op.
|
||||
func (d *taskDispatcher) Suspend(id int64) {
|
||||
// setCallWaits applies a durable snapshot without accumulating duplicate events.
|
||||
func (d *taskDispatcher) setCallWaits(hold *taskHold, waits int) {
|
||||
d.mu.Lock()
|
||||
hold := d.holds[id]
|
||||
if hold == nil || hold.finished {
|
||||
if d.holds[hold.id] != hold || hold.finished {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
hold.waits++
|
||||
if hold.waits == 1 {
|
||||
previous := hold.waits
|
||||
hold.waits = waits
|
||||
if waits > 0 {
|
||||
d.giveBackLocked(hold)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
d.slotsFree.Broadcast()
|
||||
}
|
||||
|
||||
// Resume takes a run's capacity back when one of its waits ends. Only the last
|
||||
// wait does: a run waiting on callees from parallel branches keeps nothing
|
||||
// until all of them are done. An unknown task, one whose run has returned, and
|
||||
// one that never suspended are all no-ops.
|
||||
func (d *taskDispatcher) Resume(id int64) {
|
||||
d.mu.Lock()
|
||||
hold := d.holds[id]
|
||||
if hold == nil || hold.finished || hold.waits == 0 {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
if waits > 0 {
|
||||
d.slotsFree.Broadcast()
|
||||
} else if previous > 0 {
|
||||
// Reacquiring may block behind another task; the caller keeps running.
|
||||
go d.reacquire(hold)
|
||||
}
|
||||
hold.waits--
|
||||
if hold.waits > 0 {
|
||||
d.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.mu.Unlock()
|
||||
// The two waits below can block, and neither may hold up the bus
|
||||
// subscriber calling this or the resumed run, which is already executing
|
||||
// inside its container: the execution cap can be exceeded by at most the
|
||||
// number of simultaneous resumes, and re-converges as tasks finish.
|
||||
go d.reacquire(hold)
|
||||
}
|
||||
|
||||
// reacquire returns a resumed run's capacity: its place in its repo's chain
|
||||
@@ -1176,7 +1156,7 @@ func (d *taskDispatcher) Resume(id int64) {
|
||||
func (d *taskDispatcher) reacquire(hold *taskHold) {
|
||||
d.mu.Lock()
|
||||
if hold.finished || hold.waits > 0 {
|
||||
// The run returned, or started waiting again, after the Resume that
|
||||
// The run returned, or started waiting again, after the reconciliation that
|
||||
// queued this: it needs no capacity.
|
||||
d.mu.Unlock()
|
||||
return
|
||||
@@ -1593,7 +1573,7 @@ func (d *Daemon) reconcilePRs(ctx context.Context) {
|
||||
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
|
||||
// A task off the NATS stream only has an id once the store has claimed its
|
||||
// row, inside this call, so its capacity is named here.
|
||||
hold.name(task.ID)
|
||||
hold.name(task.ID, task.Attempt)
|
||||
// The workflow may name the identity the run acts as, which everything
|
||||
// below is chosen by, so it is settled before anything else.
|
||||
if !d.adoptWorkflowIdentity(ctx, task) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"runtime"
|
||||
"sync"
|
||||
@@ -12,8 +11,10 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/taskstate"
|
||||
)
|
||||
|
||||
// runDeadline bounds a wait for something that must happen. Only a broken
|
||||
@@ -139,12 +140,25 @@ func slotHolders(d *taskDispatcher) int {
|
||||
return held
|
||||
}
|
||||
|
||||
func changeCallWaits(d *taskDispatcher, delta int, id int64) {
|
||||
d.mu.Lock()
|
||||
hold := d.holds[id]
|
||||
waits := 0
|
||||
if hold != nil {
|
||||
waits = max(0, hold.waits+delta)
|
||||
}
|
||||
d.mu.Unlock()
|
||||
if hold != nil {
|
||||
d.setCallWaits(hold, waits)
|
||||
}
|
||||
}
|
||||
|
||||
// submitRun queues a task whose process reports its start on started and then
|
||||
// blocks until released. Naming the run's capacity is what the daemon's own
|
||||
// process does, where the task's id first exists.
|
||||
func submitRun(d *taskDispatcher, id int64, repo string, started chan<- int64, released <-chan struct{}) {
|
||||
d.Submit(context.Background(), &workflow.Task{ID: id, Owner: "acme", Repo: repo}, func(_ context.Context, task *workflow.Task, hold *taskHold) {
|
||||
hold.name(task.ID)
|
||||
hold.name(task.ID, task.Attempt)
|
||||
started <- task.ID
|
||||
<-released
|
||||
})
|
||||
@@ -164,11 +178,11 @@ func TestSuspendReleasesCapacityForTheTaskBehindIt(t *testing.T) {
|
||||
submitRun(d, 2, "app", started, b)
|
||||
awaitNoStart(t, started, "while the waiting run held the only slot and the repo's place")
|
||||
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
awaitStart(t, started, 2)
|
||||
close(b)
|
||||
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
|
||||
|
||||
submitRun(d, 3, "app", started, c)
|
||||
@@ -193,11 +207,11 @@ func TestResumedRunKeepsItsPlaceInTheReposChain(t *testing.T) {
|
||||
submitRun(d, 2, "app", started, b)
|
||||
awaitNoStart(t, started, "while the repo's place was another run's")
|
||||
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
awaitStart(t, started, 2)
|
||||
close(b)
|
||||
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not take its slot and its place back", func() bool { return holdsCapacity(d, 1) })
|
||||
submitRun(d, 3, "app", started, c)
|
||||
awaitNoStart(t, started, "while the resumed run held the repo's place, with slots free")
|
||||
@@ -217,8 +231,8 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
|
||||
|
||||
submitRun(d, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
d.Suspend(1)
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
|
||||
// The callee queued behind it runs, and returns.
|
||||
submitRun(d, 2, "app", started, b)
|
||||
@@ -228,11 +242,11 @@ func TestResumeTakesCapacityBackOnlyAfterTheLastWait(t *testing.T) {
|
||||
|
||||
// One wait outstanding: the run keeps no capacity while its other callee
|
||||
// runs.
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitNoState(t, "task 1 took capacity back with a callee still running", func() bool { return holdsCapacity(d, 1) })
|
||||
|
||||
// With the last wait done it takes capacity back.
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
|
||||
|
||||
close(a)
|
||||
@@ -253,13 +267,13 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
|
||||
|
||||
submitRun(d, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
|
||||
// X takes the repo's place while A waits on its callee, so A's first
|
||||
// re-acquire queues its next place behind X.
|
||||
submitRun(d, 2, "app", started, x)
|
||||
awaitStart(t, started, 2)
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not queue its place back", func() bool {
|
||||
_, place, _ := capacity(d, 1)
|
||||
return place
|
||||
@@ -268,8 +282,8 @@ func TestResumeTwiceDoesNotCountASecondSlot(t *testing.T) {
|
||||
// That callee ends and the run starts its next call at once: the place the
|
||||
// first re-acquire claimed is given back with it, and the second re-acquire
|
||||
// takes a slot immediately.
|
||||
d.Suspend(1)
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not take its slot back", func() bool { return holdsCapacity(d, 1) })
|
||||
|
||||
// X leaves, so the first re-acquire can take a slot as well.
|
||||
@@ -308,13 +322,13 @@ func TestSecondReacquireDoesNotQueueASecondPlace(t *testing.T) {
|
||||
|
||||
submitRun(d, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
|
||||
// X holds the repo's place, so the run's re-acquire queues behind it and
|
||||
// stays there.
|
||||
submitRun(d, 2, "app", started, x)
|
||||
awaitStart(t, started, 2)
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not queue its place back", func() bool {
|
||||
_, place, _ := capacity(d, 1)
|
||||
return place
|
||||
@@ -363,14 +377,14 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
|
||||
|
||||
submitRun(d, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
d.Suspend(1)
|
||||
changeCallWaits(d, 1, 1)
|
||||
|
||||
submitRun(d, 2, "app", started, b)
|
||||
awaitStart(t, started, 2)
|
||||
|
||||
// The resume is queued for a slot and a place in the chain, and overtaken by
|
||||
// the run returning before either comes free.
|
||||
d.Resume(1)
|
||||
changeCallWaits(d, -1, 1)
|
||||
awaitState(t, "task 1 did not take its place in the chain back", func() bool {
|
||||
_, place, _ := capacity(d, 1)
|
||||
return place
|
||||
@@ -394,49 +408,7 @@ func TestResumeAfterTheRunReturnedLeavesNoCapacity(t *testing.T) {
|
||||
close(c)
|
||||
|
||||
// A Resume arriving after the run returned is a no-op.
|
||||
d.Resume(1)
|
||||
d.Wait()
|
||||
}
|
||||
|
||||
// callEvent builds one call step's event as the daemon receives it: the event
|
||||
// the container's workflow engine published, JSON-encoded and decoded again by
|
||||
// the agent-event bridge, so the wait flag arrives as a decoded bool.
|
||||
func callEvent(t *testing.T, kind string, taskID int64, wait bool) events.Event {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(events.Event{Kind: kind, TaskID: taskID, Data: map[string]any{"wait": wait}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e events.Event
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// Only a call the caller waits on releases its capacity, and the call
|
||||
// finishing takes it back.
|
||||
func TestCallWaitEventsReleaseCapacity(t *testing.T) {
|
||||
d := newTaskDispatcher(1, nil)
|
||||
started := make(chan int64, 4)
|
||||
a, b := make(chan struct{}), make(chan struct{})
|
||||
|
||||
submitRun(d, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
submitRun(d, 2, "app", started, b)
|
||||
|
||||
// A call that does not wait is not waited on: nothing is released.
|
||||
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, false))
|
||||
awaitNoStart(t, started, "while the call that started does not wait")
|
||||
|
||||
applyCallWait(d, callEvent(t, events.KindWorkflowCallStarted, 1, true))
|
||||
awaitStart(t, started, 2)
|
||||
close(b)
|
||||
|
||||
applyCallWait(d, callEvent(t, events.KindWorkflowCallFinished, 1, false))
|
||||
awaitState(t, "task 1 did not take its capacity back", func() bool { return holdsCapacity(d, 1) })
|
||||
|
||||
close(a)
|
||||
changeCallWaits(d, -1, 1)
|
||||
d.Wait()
|
||||
}
|
||||
|
||||
@@ -447,6 +419,65 @@ func TestCallWaitsWithoutABus(t *testing.T) {
|
||||
d.WaitForTasks()
|
||||
}
|
||||
|
||||
type callWaitStore struct {
|
||||
storecontract.TaskStore
|
||||
mu sync.Mutex
|
||||
steps []task.StepExecution
|
||||
}
|
||||
|
||||
func (s *callWaitStore) ListSteps(_ context.Context, id int64, _ int) ([]task.StepExecution, error) {
|
||||
if id != 1 {
|
||||
return nil, nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]task.StepExecution(nil), s.steps...), nil
|
||||
}
|
||||
|
||||
func TestCallWaitsRecoverDroppedStart(t *testing.T) {
|
||||
store := &callWaitStore{steps: []task.StepExecution{
|
||||
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 2},
|
||||
{Kind: task.StepKindCall, Status: taskstate.StepRunning, CalledExecutionID: 3},
|
||||
}}
|
||||
cfg := config.Config{}
|
||||
cfg.Containers.MaxConcurrency = 1
|
||||
d := &Daemon{Cfg: config.NewHolder(cfg), Store: store, Bus: events.NewBus(), Log: slog.New(slog.DiscardHandler)}
|
||||
dispatcher := d.taskDispatcher()
|
||||
started := make(chan int64, 2)
|
||||
a, b := make(chan struct{}), make(chan struct{})
|
||||
defer func() { close(a); dispatcher.Wait() }()
|
||||
defer func() {
|
||||
if b != nil {
|
||||
close(b)
|
||||
}
|
||||
}()
|
||||
submitRun(dispatcher, 1, "app", started, a)
|
||||
awaitStart(t, started, 1)
|
||||
submitRun(dispatcher, 2, "app", started, b)
|
||||
// The live start event was dropped; only the recorded call remains.
|
||||
d.watchCallWaits(t.Context())
|
||||
awaitStart(t, started, 2)
|
||||
close(b)
|
||||
b = nil
|
||||
// Repeated reads must not accumulate waits; a partial finish stays suspended.
|
||||
store.mu.Lock()
|
||||
store.steps[0].Status = taskstate.StepSucceeded
|
||||
store.mu.Unlock()
|
||||
awaitState(t, "remaining call was not reconciled", func() bool {
|
||||
dispatcher.mu.Lock()
|
||||
defer dispatcher.mu.Unlock()
|
||||
return dispatcher.holds[1].waits == 1
|
||||
})
|
||||
if holdsCapacity(dispatcher, 1) {
|
||||
t.Fatal("parent reacquired capacity with a call still open")
|
||||
}
|
||||
// The finish event is also missing; the durable close restores capacity.
|
||||
store.mu.Lock()
|
||||
store.steps[1].Status = taskstate.StepSucceeded
|
||||
store.mu.Unlock()
|
||||
awaitState(t, "parent did not recover its capacity", func() bool { return holdsCapacity(dispatcher, 1) })
|
||||
}
|
||||
|
||||
// stubTasks is a task bus with an empty stream, so a drain pass falls through
|
||||
// to the store's ClaimNext.
|
||||
type stubTasks struct{}
|
||||
|
||||
Reference in new issue
Block a user