243 lines
7.5 KiB
Go
243 lines
7.5 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"log/slog"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// fakeWork is an in-memory worker store with lease-fenced writes, used to
|
|
// drive the engine through a lost lease and a conflicting step write.
|
|
type fakeWork struct {
|
|
mu sync.Mutex
|
|
// leaseLost makes every fenced write fail as if another worker held the run.
|
|
leaseLost bool
|
|
// recordedElsewhere is a step another lease holder already recorded; it
|
|
// becomes visible once this worker tries to write the same position.
|
|
recordedElsewhere *store.Step
|
|
// rejectFrom, when set, makes every write from that many recorded steps
|
|
// on fail as a lost lease: the worker was killed there.
|
|
rejectFrom *int
|
|
steps []store.Step
|
|
finished []string
|
|
errors []string
|
|
details []string
|
|
results []jsontext.Value
|
|
// bodies are the stored workflows by name. A workflow not listed is a
|
|
// single notify step.
|
|
bodies map[string]string
|
|
// started is every run started for another, once per start key.
|
|
started []store.NewRun
|
|
}
|
|
|
|
func (f *fakeWork) Claim(context.Context, string, uuid.UUID, time.Duration) (store.Run, bool, error) {
|
|
return store.Run{}, false, nil
|
|
}
|
|
|
|
func (f *fakeWork) RenewLease(context.Context, uuid.UUID, uuid.UUID, time.Duration) (bool, error) {
|
|
return true, nil
|
|
}
|
|
|
|
func (f *fakeWork) Steps(context.Context, uuid.UUID) ([]store.Step, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]store.Step(nil), f.steps...), nil
|
|
}
|
|
|
|
func (f *fakeWork) RecordStep(ctx context.Context, _ uuid.UUID, step store.Step) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
// A database write fails on a cancelled context.
|
|
if err := ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
if f.leaseLost || (f.rejectFrom != nil && len(f.steps) >= *f.rejectFrom) {
|
|
return store.ErrLeaseLost
|
|
}
|
|
if f.recordedElsewhere != nil && f.recordedElsewhere.Seq == step.Seq {
|
|
f.steps = append(f.steps, *f.recordedElsewhere)
|
|
return store.ErrStepExists
|
|
}
|
|
f.steps = append(f.steps, step)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, string) error {
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, e store.Ended) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.leaseLost {
|
|
return store.ErrLeaseLost
|
|
}
|
|
f.finished = append(f.finished, e.State)
|
|
f.errors = append(f.errors, e.Error)
|
|
f.details = append(f.details, e.Detail)
|
|
f.results = append(f.results, e.Result)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeWork) Signalled(context.Context, uuid.UUID, string) (jsontext.Value, bool, error) {
|
|
return nil, false, nil
|
|
}
|
|
|
|
func (f *fakeWork) Workflow(_ context.Context, _, name string, _ int) (store.Workflow, error) {
|
|
body, ok := f.bodies[name]
|
|
if !ok {
|
|
body = "steps:\n - type: run.notify\n"
|
|
}
|
|
return store.Workflow{Name: name, Version: 1, Body: body}, nil
|
|
}
|
|
|
|
// StartFor starts a run once per start key, as the store does.
|
|
func (f *fakeWork) StartFor(_ context.Context, _, _ uuid.UUID, n store.NewRun) (uuid.UUID, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.leaseLost {
|
|
return uuid.Nil, store.ErrLeaseLost
|
|
}
|
|
for _, started := range f.started {
|
|
if started.StartKey == n.StartKey {
|
|
return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil
|
|
}
|
|
}
|
|
f.started = append(f.started, n)
|
|
return uuid.NewSHA1(uuid.NameSpaceURL, []byte(n.StartKey)), nil
|
|
}
|
|
|
|
func (f *fakeWork) CancelRequested(context.Context, uuid.UUID) (bool, error) {
|
|
return false, nil
|
|
}
|
|
|
|
// TestLeaseFencesStepWrites guards 2.2's guarantee that correctness rests on
|
|
// fenced writes: a worker that lost its lease stops before running another
|
|
// step (which could repeat an effect), and a step another holder already
|
|
// recorded wins over this worker's own result.
|
|
func TestLeaseFencesStepWrites(t *testing.T) {
|
|
theirs := jsontext.Value(`"theirs"`)
|
|
tests := []struct {
|
|
name string
|
|
work *fakeWork
|
|
wantSteps int
|
|
wantEffects int
|
|
wantFinished []string
|
|
wantResult string
|
|
}{
|
|
{
|
|
name: "lost lease stops the run before its next step",
|
|
work: &fakeWork{leaseLost: true},
|
|
wantSteps: 0,
|
|
wantEffects: 0,
|
|
wantFinished: nil,
|
|
},
|
|
{
|
|
name: "a step recorded by another holder replaces this worker's result",
|
|
work: &fakeWork{recordedElsewhere: &store.Step{
|
|
Seq: 0, Name: "probe", Outcome: store.OutcomeDone, Output: theirs,
|
|
}},
|
|
wantSteps: 2,
|
|
wantEffects: 1,
|
|
wantFinished: []string{store.StateSucceeded},
|
|
wantResult: `"theirs"`,
|
|
},
|
|
{
|
|
name: "a held lease records the steps and finishes the run",
|
|
work: &fakeWork{},
|
|
wantSteps: 2,
|
|
wantEffects: 1,
|
|
wantFinished: []string{store.StateSucceeded},
|
|
wantResult: `"mine"`,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{})
|
|
effects := 0
|
|
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
|
out, err := rc.Step("probe", func(context.Context, string) (string, error) {
|
|
return "mine", nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := rc.Step("effect", func(context.Context, string) (bool, error) {
|
|
effects++
|
|
return true, nil
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
return json.Marshal(out)
|
|
})
|
|
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Version: 1, Input: jsontext.Value(`null`)}
|
|
engine.execute(t.Context(), run, uuid.New())
|
|
|
|
if got := len(tt.work.steps); got != tt.wantSteps {
|
|
t.Fatalf("steps recorded = %d, want %d", got, tt.wantSteps)
|
|
}
|
|
if effects != tt.wantEffects {
|
|
t.Fatalf("effect step ran %d times, want %d", effects, tt.wantEffects)
|
|
}
|
|
if len(tt.work.finished) != len(tt.wantFinished) {
|
|
t.Fatalf("finished = %v, want %v", tt.work.finished, tt.wantFinished)
|
|
}
|
|
if len(tt.wantFinished) > 0 && string(tt.work.results[0]) != tt.wantResult {
|
|
t.Fatalf("result = %s, want %s", tt.work.results[0], tt.wantResult)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestStepValueIsTheSameLiveAndOnReplay reproduces a resumed agent diverging
|
|
// from its journal: a pinned value carried raw JSON (a tool schema), the live
|
|
// attempt used it as written while the resume used it as stored, and the
|
|
// request digests differed. A step must hand its workflow the value as
|
|
// recorded on both paths.
|
|
func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
|
type pinned struct {
|
|
Schema jsontext.Value `json:"schema"`
|
|
}
|
|
var seen []string
|
|
register := func(e *Engine) {
|
|
useWorkflow(e, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
|
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
|
|
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
seen = append(seen, string(got.Schema))
|
|
return jsontext.Value(`null`), nil
|
|
})
|
|
}
|
|
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Version: 1, Input: jsontext.Value(`null`)}
|
|
|
|
live := &fakeWork{}
|
|
e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{})
|
|
register(e)
|
|
e.execute(t.Context(), run, uuid.New())
|
|
|
|
replayed := &fakeWork{steps: live.steps}
|
|
e = newEngine(nil, replayed, slog.New(slog.DiscardHandler), Options{})
|
|
register(e)
|
|
e.execute(t.Context(), run, uuid.New())
|
|
|
|
if len(seen) != 2 || seen[0] != seen[1] {
|
|
t.Fatalf("live and replayed values differ: %q", seen)
|
|
}
|
|
}
|
|
|
|
// useWorkflow makes e run wf for every run, in place of a stored document.
|
|
func useWorkflow(e *Engine, wf Workflow) {
|
|
e.load = func(context.Context, store.Run) (Workflow, error) { return wf, nil }
|
|
}
|