fix(runs): a step hands back its value as recorded, so a resumed agent doesn't diverge from its journal (#1977)

Killing the binary mid-investigation and restarting it failed the run: the
first model request after the resume didn't match the recorded one. The tool
schemas pinned in a step reached the live attempt as written but the resume
as stored, and both the v2 encoder (compacting raw JSON) and jsonb (reordering
keys) rewrite them. Step now returns the decoded recorded value on the live
path too, and runs.steps.output is json, which keeps the text as written.
This commit is contained in:
sam committed 2026-11-16 00:36:19 +11:00
1 parent 0335872701
commit 4877362223
3 files changed
+54 -3

No files matched your search

@@ -0,0 +1,8 @@
-- +goose Up
-- A step's output must replay byte for byte: jsonb reorders keys and drops
-- whitespace, so a pinned value (a tool schema) read back differed from the
-- one a live attempt used, and a resumed agent diverged from its journal.
ALTER TABLE runs.steps ALTER COLUMN output TYPE json USING output::json;
-- +goose Down
ALTER TABLE runs.steps ALTER COLUMN output TYPE jsonb USING output::jsonb;
+7 -3
View File
@@ -57,8 +57,9 @@ func (rc *Context) RunID() uuid.UUID { return rc.run.ID }
func (rc *Context) OrgID() string { return rc.run.OrgID }
// Step runs fn once for this position in the workflow and saves its result.
// On replay it returns the saved result without calling fn. fn receives an
// idempotency key unique to this run and step, for connectors to honour.
// It returns the result as saved, and on replay returns it without calling fn,
// so both paths see the same value. fn receives an idempotency key unique to
// this run and step, for connectors to honour.
func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) {
var zero T
seq := rc.advance()
@@ -81,7 +82,10 @@ func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key str
if step.Outcome != store.OutcomeDone {
return zero, err
}
return out, nil
// Hand back the value as recorded, not as fn returned it: encoding can
// rewrite it (raw JSON is compacted), and a resumed run must see exactly
// what this one sees.
return replay[T](step, name)
}
// Sleep suspends the run durably until d has passed since the first time this
+39
View File
@@ -163,3 +163,42 @@ func TestLeaseFencesStepWrites(t *testing.T) {
})
}
}
// TestStepValueIsTheSameLiveAndOnReplay reproduces a resumed agent diverging
// from its journal: a pinned value carried raw JSON (a tool schema), the live
// attempt used it as written while the resume used it as stored, and the
// request digests differed. A step must hand its workflow the value as
// recorded on both paths.
func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
type pinned struct {
Schema jsontext.Value `json:"schema"`
}
var seen []string
register := func(e *Engine) {
e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
})
if err != nil {
return nil, err
}
seen = append(seen, string(got.Schema))
return jsontext.Value(`null`), nil
})
}
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)}
live := &fakeWork{}
e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{})
register(e)
e.execute(t.Context(), run, uuid.New())
replayed := &fakeWork{steps: live.steps}
e = newEngine(nil, replayed, slog.New(slog.DiscardHandler), Options{})
register(e)
e.execute(t.Context(), run, uuid.New())
if len(seen) != 2 || seen[0] != seen[1] {
t.Fatalf("live and replayed values differ: %q", seen)
}
}