From 48773622234d43ff3012f2230a317ecbdaa54916 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 00:36:19 +1100 Subject: [PATCH] fix(runs): a step hands back its value as recorded, so a resumed agent doesn't diverge from its journal (#1977) Killing the binary mid-investigation and restarting it failed the run: the first model request after the resume didn't match the recorded one. The tool schemas pinned in a step reached the live attempt as written but the resume as stored, and both the v2 encoder (compacting raw JSON) and jsonb (reordering keys) rewrite them. Step now returns the decoded recorded value on the live path too, and runs.steps.output is json, which keeps the text as written. --- .../pg/migrations/00007_steps_output_json.sql | 8 ++++ modules/runs/context.go | 10 +++-- modules/runs/runs_test.go | 39 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 internal/platform/pg/migrations/00007_steps_output_json.sql diff --git a/internal/platform/pg/migrations/00007_steps_output_json.sql b/internal/platform/pg/migrations/00007_steps_output_json.sql new file mode 100644 index 000000000..3d40173e8 --- /dev/null +++ b/internal/platform/pg/migrations/00007_steps_output_json.sql @@ -0,0 +1,8 @@ +-- +goose Up +-- A step's output must replay byte for byte: jsonb reorders keys and drops +-- whitespace, so a pinned value (a tool schema) read back differed from the +-- one a live attempt used, and a resumed agent diverged from its journal. +ALTER TABLE runs.steps ALTER COLUMN output TYPE json USING output::json; + +-- +goose Down +ALTER TABLE runs.steps ALTER COLUMN output TYPE jsonb USING output::jsonb; diff --git a/modules/runs/context.go b/modules/runs/context.go index 77a319ee3..b9651fa57 100644 --- a/modules/runs/context.go +++ b/modules/runs/context.go @@ -57,8 +57,9 @@ func (rc *Context) RunID() uuid.UUID { return rc.run.ID } func (rc *Context) OrgID() string { return rc.run.OrgID } // Step runs fn once for this position in the workflow and saves its result. -// On replay it returns the saved result without calling fn. fn receives an -// idempotency key unique to this run and step, for connectors to honour. +// It returns the result as saved, and on replay returns it without calling fn, +// so both paths see the same value. fn receives an idempotency key unique to +// this run and step, for connectors to honour. func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key string) (T, error)) (T, error) { var zero T seq := rc.advance() @@ -81,7 +82,10 @@ func (rc *Context) Step[T any](name string, fn func(ctx context.Context, key str if step.Outcome != store.OutcomeDone { return zero, err } - return out, nil + // Hand back the value as recorded, not as fn returned it: encoding can + // rewrite it (raw JSON is compacted), and a resumed run must see exactly + // what this one sees. + return replay[T](step, name) } // Sleep suspends the run durably until d has passed since the first time this diff --git a/modules/runs/runs_test.go b/modules/runs/runs_test.go index 32e1c20b5..2bc93486d 100644 --- a/modules/runs/runs_test.go +++ b/modules/runs/runs_test.go @@ -163,3 +163,42 @@ func TestLeaseFencesStepWrites(t *testing.T) { }) } } + +// TestStepValueIsTheSameLiveAndOnReplay reproduces a resumed agent diverging +// from its journal: a pinned value carried raw JSON (a tool schema), the live +// attempt used it as written while the resume used it as stored, and the +// request digests differed. A step must hand its workflow the value as +// recorded on both paths. +func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) { + type pinned struct { + Schema jsontext.Value `json:"schema"` + } + var seen []string + register := func(e *Engine) { + e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { + got, err := rc.Step("pin", func(context.Context, string) (pinned, error) { + return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil + }) + if err != nil { + return nil, err + } + seen = append(seen, string(got.Schema)) + return jsontext.Value(`null`), nil + }) + } + run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)} + + live := &fakeWork{} + e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{}) + register(e) + e.execute(t.Context(), run, uuid.New()) + + replayed := &fakeWork{steps: live.steps} + e = newEngine(nil, replayed, slog.New(slog.DiscardHandler), Options{}) + register(e) + e.execute(t.Context(), run, uuid.New()) + + if len(seen) != 2 || seen[0] != seen[1] { + t.Fatalf("live and replayed values differ: %q", seen) + } +}