feat(runs): a run is pinned to the workflow version it started with and fails clearly when it changed (archie #2014)

This commit is contained in:
sam committed 2026-11-16 05:04:30 +11:00
1 parent a0385a56c7
commit 400699cfbb
8 files changed
+99 -28

No files matched your search

+1 -1
View File
@@ -94,7 +94,7 @@ func Run(ctx context.Context, configPath string) error {
allow.Routes(mux, guard)
classes.Routes(mux, guard, allow)
remembered := memory.NewModule(db, orgID)
engine.Register("investigate", runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above
engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above
// An answer a channel delivers is an event, and settles its request.
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
@@ -0,0 +1,6 @@
-- +goose Up
-- Runs that exist before versions were declared started under version 1.
ALTER TABLE runs.runs ADD COLUMN workflow_version integer NOT NULL DEFAULT 1;
-- +goose Down
ALTER TABLE runs.runs DROP COLUMN workflow_version;
+2 -2
View File
@@ -118,7 +118,7 @@ func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
t.Helper()
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
engine.Register("agent", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
engine.Register("agent", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
return nil, err
}
@@ -141,7 +141,7 @@ func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provi
}
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
})
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Input: jsontext.Value(`null`)}
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(ctx, run, uuid.New())
}
+13 -10
View File
@@ -41,9 +41,11 @@ var (
// Run is a claimed run.
type Run struct {
ID uuid.UUID `db:"id"`
OrgID string `db:"org_id"`
Workflow string `db:"workflow"`
ID uuid.UUID `db:"id"`
OrgID string `db:"org_id"`
Workflow string `db:"workflow"`
// Version is the workflow version the run started under.
Version int `db:"workflow_version"`
Input jsontext.Value `db:"input"`
Attempts int `db:"attempts"`
}
@@ -70,14 +72,14 @@ func New(db *sqlx.DB) *Store {
// Start inserts a queued run. A run with the same organisation and start key
// is not inserted again; its id is returned instead.
func (s *Store) Start(ctx context.Context, orgID, workflow, startKey string, input jsontext.Value) (uuid.UUID, error) {
func (s *Store) Start(ctx context.Context, orgID, workflow string, version int, startKey string, input jsontext.Value) (uuid.UUID, error) {
key := sql.NullString{String: startKey, Valid: startKey != ""}
var id uuid.UUID
err := s.db.GetContext(ctx, &id, `
INSERT INTO runs.runs (id, org_id, workflow, start_key, input, state)
VALUES ($1, $2, $3, $4, $5, 'queued')
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state)
VALUES ($1, $2, $3, $4, $5, $6, 'queued')
ON CONFLICT (org_id, start_key) DO NOTHING
RETURNING id`, uuid.New(), orgID, workflow, key, input)
RETURNING id`, uuid.New(), orgID, workflow, version, key, input)
if errors.Is(err, sql.ErrNoRows) && key.Valid {
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey)
}
@@ -105,7 +107,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti
ORDER BY created_at
FOR UPDATE SKIP LOCKED
LIMIT 1)
RETURNING r.id, r.org_id, r.workflow, r.input, r.attempts`, token, owner, ttl.Seconds())
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts`, token, owner, ttl.Seconds())
if errors.Is(err, sql.ErrNoRows) {
return Run{}, false, nil
}
@@ -253,6 +255,7 @@ func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
type Summary struct {
ID uuid.UUID `db:"id"`
Workflow string `db:"workflow"`
Version int `db:"workflow_version"`
State string `db:"state"`
Attempts int `db:"attempts"`
Error sql.NullString `db:"error"`
@@ -279,7 +282,7 @@ type Recorded struct {
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
var out []Summary
if err := s.db.SelectContext(ctx, &out, `
SELECT id, workflow, state, attempts, error, created_at, finished_at FROM runs.runs
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at FROM runs.runs
WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list runs: %w", err)
}
@@ -291,7 +294,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
var d Detail
err := s.db.GetContext(ctx, &d, `
SELECT id, workflow, state, attempts, error, created_at, finished_at, input,
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, input,
coalesce(result, 'null') AS result
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
if errors.Is(err, sql.ErrNoRows) {
+26 -9
View File
@@ -24,6 +24,12 @@ import (
// lookups belong inside steps.
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
// registered is a workflow with the version its code declares.
type registered struct {
version int
run Workflow
}
// StartRequest names the workflow to run and the organisation it runs for.
// A non-empty Key makes the start idempotent within the organisation.
type StartRequest struct {
@@ -60,7 +66,7 @@ type Engine struct {
log *slog.Logger
opts Options
mu sync.RWMutex
workflows map[string]Workflow
workflows map[string]registered
}
// New returns an engine on db.
@@ -82,23 +88,30 @@ func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options
if opts.Owner == "" {
opts.Owner = "archie"
}
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]Workflow{}}
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]registered{}}
}
// Register makes a workflow available to runs by name.
func (e *Engine) Register(name string, wf Workflow) {
// Register makes a workflow available to runs by name. Raise version whenever
// a change to wf would alter the steps an in-flight run replays: a run only
// executes under the version it started with.
func (e *Engine) Register(name string, version int, wf Workflow) {
e.mu.Lock()
defer e.mu.Unlock()
e.workflows[name] = wf
e.workflows[name] = registered{version: version, run: wf}
}
// Start queues a run and returns its id.
// Start queues a run under the registered version of its workflow and returns
// its id.
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
wf, ok := e.workflow(req.Workflow)
if !ok {
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
}
input := req.Input
if len(input) == 0 {
input = jsontext.Value("null")
}
return e.db.Start(ctx, req.OrgID, req.Workflow, req.Key, input)
return e.db.Start(ctx, req.OrgID, req.Workflow, wf.version, req.Key, input)
}
// Signal delivers a payload to a run waiting on key, now or later.
@@ -188,14 +201,18 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
if !ok {
return nil, fmt.Errorf("no workflow named %q", run.Workflow)
}
if run.Version != wf.version {
return nil, fmt.Errorf("run started under version %d of workflow %q, this build runs version %d; start a new run",
run.Version, run.Workflow, wf.version)
}
steps, err := e.work.Steps(ctx, run.ID)
if err != nil {
return nil, err
}
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
return wf.run(newContext(ctx, e.work, run, token, steps), run.Input)
}
func (e *Engine) workflow(name string) (Workflow, bool) {
func (e *Engine) workflow(name string) (registered, bool) {
e.mu.RLock()
defer e.mu.RUnlock()
wf, ok := e.workflows[name]
+48 -5
View File
@@ -5,6 +5,7 @@ import (
"encoding/json/jsontext"
json "encoding/json/v2"
"log/slog"
"strings"
"sync"
"testing"
"time"
@@ -28,6 +29,7 @@ type fakeWork struct {
rejectFrom *int
steps []store.Step
finished []string
errors []string
results []jsontext.Value
}
@@ -67,13 +69,14 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st
return nil
}
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, _ string) error {
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseLost {
return store.ErrLeaseLost
}
f.finished = append(f.finished, state)
f.errors = append(f.errors, errText)
f.results = append(f.results, result)
return nil
}
@@ -130,7 +133,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{})
effects := 0
engine.Register("probe", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
engine.Register("probe", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
out, err := rc.Step("probe", func(context.Context, string) (string, error) {
return "mine", nil
})
@@ -145,7 +148,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
}
return json.Marshal(out)
})
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Input: jsontext.Value(`null`)}
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(t.Context(), run, uuid.New())
if got := len(tt.work.steps); got != tt.wantSteps {
@@ -175,7 +178,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
}
var seen []string
register := func(e *Engine) {
e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
e.Register("pin", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
})
@@ -186,7 +189,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
return jsontext.Value(`null`), nil
})
}
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)}
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Version: 1, Input: jsontext.Value(`null`)}
live := &fakeWork{}
e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{})
@@ -202,3 +205,43 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
t.Fatalf("live and replayed values differ: %q", seen)
}
}
// TestRunExecutesOnlyUnderItsStartVersion guards workflow versioning: a run
// resumed after its workflow's version changed must fail naming both versions
// and run none of the new code, never replay old steps through new logic.
func TestRunExecutesOnlyUnderItsStartVersion(t *testing.T) {
tests := []struct {
name string
runVersion int
wantState string
wantRan bool
wantErrPart string
}{
{name: "same version runs", runVersion: 2, wantState: store.StateSucceeded, wantRan: true},
{name: "older run fails naming both versions", runVersion: 1, wantState: store.StateFailed, wantErrPart: "version 1"},
{name: "newer run fails when the build is behind", runVersion: 3, wantState: store.StateFailed, wantErrPart: "version 2"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
work := &fakeWork{}
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
ran := false
engine.Register("versioned", 2, func(*Context, jsontext.Value) (jsontext.Value, error) {
ran = true
return jsontext.Value(`null`), nil
})
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "versioned", Version: tt.runVersion, Input: jsontext.Value(`null`)}
engine.execute(t.Context(), run, uuid.New())
if ran != tt.wantRan {
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
}
if len(work.finished) != 1 || work.finished[0] != tt.wantState {
t.Fatalf("finished = %v, want [%s]", work.finished, tt.wantState)
}
if tt.wantErrPart != "" && !strings.Contains(work.errors[0], tt.wantErrPart) {
t.Fatalf("error %q does not name %q", work.errors[0], tt.wantErrPart)
}
})
}
}
+2 -1
View File
@@ -40,6 +40,7 @@ type Origins interface {
type RunView struct {
ID uuid.UUID `json:"id"`
Workflow string `json:"workflow"`
Version int `json:"workflow_version"`
State string `json:"state"`
Attempts int `json:"attempts"`
Error string `json:"error,omitzero"`
@@ -292,7 +293,7 @@ func effectOf(s store.Recorded) *Effect {
func view(s store.Summary, result jsontext.Value) RunView {
v := RunView{
ID: s.ID, Workflow: s.Workflow, State: s.State, Attempts: s.Attempts,
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result,
}
if s.FinishedAt.Valid {
+1
View File
@@ -44,6 +44,7 @@ export interface Usage {
export interface Run {
id: string;
workflow: string;
workflow_version: number;
state: "queued" | "running" | "waiting" | "succeeded" | "failed" | "cancelled";
attempts: number;
error?: string;