feat(runs): a run is pinned to the workflow version it started with and fails clearly when it changed (archie #2014)

This commit is contained in:
sam committed 2026-11-16 05:04:30 +11:00
1 parent a0385a56c7
commit 400699cfbb
8 files changed
+99 -28

No files matched your search

+1 -1
View File
@@ -94,7 +94,7 @@ func Run(ctx context.Context, configPath string) error {
allow.Routes(mux, guard) allow.Routes(mux, guard)
classes.Routes(mux, guard, allow) classes.Routes(mux, guard, allow)
remembered := memory.NewModule(db, orgID) remembered := memory.NewModule(db, orgID)
engine.Register("investigate", runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above
// An answer a channel delivers is an event, and settles its request. // An answer a channel delivers is an event, and settles its request.
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow) hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
@@ -0,0 +1,6 @@
-- +goose Up
-- Runs that exist before versions were declared started under version 1.
ALTER TABLE runs.runs ADD COLUMN workflow_version integer NOT NULL DEFAULT 1;
-- +goose Down
ALTER TABLE runs.runs DROP COLUMN workflow_version;
+2 -2
View File
@@ -118,7 +118,7 @@ func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) { func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
t.Helper() t.Helper()
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{}) engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
engine.Register("agent", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { engine.Register("agent", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil { if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
return nil, err return nil, err
} }
@@ -141,7 +141,7 @@ func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provi
} }
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) }) return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
}) })
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Input: jsontext.Value(`null`)} run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(ctx, run, uuid.New()) engine.execute(ctx, run, uuid.New())
} }
+13 -10
View File
@@ -41,9 +41,11 @@ var (
// Run is a claimed run. // Run is a claimed run.
type Run struct { type Run struct {
ID uuid.UUID `db:"id"` ID uuid.UUID `db:"id"`
OrgID string `db:"org_id"` OrgID string `db:"org_id"`
Workflow string `db:"workflow"` Workflow string `db:"workflow"`
// Version is the workflow version the run started under.
Version int `db:"workflow_version"`
Input jsontext.Value `db:"input"` Input jsontext.Value `db:"input"`
Attempts int `db:"attempts"` Attempts int `db:"attempts"`
} }
@@ -70,14 +72,14 @@ func New(db *sqlx.DB) *Store {
// Start inserts a queued run. A run with the same organisation and start key // Start inserts a queued run. A run with the same organisation and start key
// is not inserted again; its id is returned instead. // is not inserted again; its id is returned instead.
func (s *Store) Start(ctx context.Context, orgID, workflow, startKey string, input jsontext.Value) (uuid.UUID, error) { func (s *Store) Start(ctx context.Context, orgID, workflow string, version int, startKey string, input jsontext.Value) (uuid.UUID, error) {
key := sql.NullString{String: startKey, Valid: startKey != ""} key := sql.NullString{String: startKey, Valid: startKey != ""}
var id uuid.UUID var id uuid.UUID
err := s.db.GetContext(ctx, &id, ` err := s.db.GetContext(ctx, &id, `
INSERT INTO runs.runs (id, org_id, workflow, start_key, input, state) INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state)
VALUES ($1, $2, $3, $4, $5, 'queued') VALUES ($1, $2, $3, $4, $5, $6, 'queued')
ON CONFLICT (org_id, start_key) DO NOTHING ON CONFLICT (org_id, start_key) DO NOTHING
RETURNING id`, uuid.New(), orgID, workflow, key, input) RETURNING id`, uuid.New(), orgID, workflow, version, key, input)
if errors.Is(err, sql.ErrNoRows) && key.Valid { if errors.Is(err, sql.ErrNoRows) && key.Valid {
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey) err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey)
} }
@@ -105,7 +107,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti
ORDER BY created_at ORDER BY created_at
FOR UPDATE SKIP LOCKED FOR UPDATE SKIP LOCKED
LIMIT 1) LIMIT 1)
RETURNING r.id, r.org_id, r.workflow, r.input, r.attempts`, token, owner, ttl.Seconds()) RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts`, token, owner, ttl.Seconds())
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
return Run{}, false, nil return Run{}, false, nil
} }
@@ -253,6 +255,7 @@ func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
type Summary struct { type Summary struct {
ID uuid.UUID `db:"id"` ID uuid.UUID `db:"id"`
Workflow string `db:"workflow"` Workflow string `db:"workflow"`
Version int `db:"workflow_version"`
State string `db:"state"` State string `db:"state"`
Attempts int `db:"attempts"` Attempts int `db:"attempts"`
Error sql.NullString `db:"error"` Error sql.NullString `db:"error"`
@@ -279,7 +282,7 @@ type Recorded struct {
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) { func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
var out []Summary var out []Summary
if err := s.db.SelectContext(ctx, &out, ` if err := s.db.SelectContext(ctx, &out, `
SELECT id, workflow, state, attempts, error, created_at, finished_at FROM runs.runs SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at FROM runs.runs
WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil { WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
return nil, fmt.Errorf("list runs: %w", err) return nil, fmt.Errorf("list runs: %w", err)
} }
@@ -291,7 +294,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) { func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
var d Detail var d Detail
err := s.db.GetContext(ctx, &d, ` err := s.db.GetContext(ctx, &d, `
SELECT id, workflow, state, attempts, error, created_at, finished_at, input, SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, input,
coalesce(result, 'null') AS result coalesce(result, 'null') AS result
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id) FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
if errors.Is(err, sql.ErrNoRows) { if errors.Is(err, sql.ErrNoRows) {
+26 -9
View File
@@ -24,6 +24,12 @@ import (
// lookups belong inside steps. // lookups belong inside steps.
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error) type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
// registered is a workflow with the version its code declares.
type registered struct {
version int
run Workflow
}
// StartRequest names the workflow to run and the organisation it runs for. // StartRequest names the workflow to run and the organisation it runs for.
// A non-empty Key makes the start idempotent within the organisation. // A non-empty Key makes the start idempotent within the organisation.
type StartRequest struct { type StartRequest struct {
@@ -60,7 +66,7 @@ type Engine struct {
log *slog.Logger log *slog.Logger
opts Options opts Options
mu sync.RWMutex mu sync.RWMutex
workflows map[string]Workflow workflows map[string]registered
} }
// New returns an engine on db. // New returns an engine on db.
@@ -82,23 +88,30 @@ func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options
if opts.Owner == "" { if opts.Owner == "" {
opts.Owner = "archie" opts.Owner = "archie"
} }
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]Workflow{}} return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]registered{}}
} }
// Register makes a workflow available to runs by name. // Register makes a workflow available to runs by name. Raise version whenever
func (e *Engine) Register(name string, wf Workflow) { // a change to wf would alter the steps an in-flight run replays: a run only
// executes under the version it started with.
func (e *Engine) Register(name string, version int, wf Workflow) {
e.mu.Lock() e.mu.Lock()
defer e.mu.Unlock() defer e.mu.Unlock()
e.workflows[name] = wf e.workflows[name] = registered{version: version, run: wf}
} }
// Start queues a run and returns its id. // Start queues a run under the registered version of its workflow and returns
// its id.
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) { func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
wf, ok := e.workflow(req.Workflow)
if !ok {
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
}
input := req.Input input := req.Input
if len(input) == 0 { if len(input) == 0 {
input = jsontext.Value("null") input = jsontext.Value("null")
} }
return e.db.Start(ctx, req.OrgID, req.Workflow, req.Key, input) return e.db.Start(ctx, req.OrgID, req.Workflow, wf.version, req.Key, input)
} }
// Signal delivers a payload to a run waiting on key, now or later. // Signal delivers a payload to a run waiting on key, now or later.
@@ -188,14 +201,18 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
if !ok { if !ok {
return nil, fmt.Errorf("no workflow named %q", run.Workflow) return nil, fmt.Errorf("no workflow named %q", run.Workflow)
} }
if run.Version != wf.version {
return nil, fmt.Errorf("run started under version %d of workflow %q, this build runs version %d; start a new run",
run.Version, run.Workflow, wf.version)
}
steps, err := e.work.Steps(ctx, run.ID) steps, err := e.work.Steps(ctx, run.ID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return wf(newContext(ctx, e.work, run, token, steps), run.Input) return wf.run(newContext(ctx, e.work, run, token, steps), run.Input)
} }
func (e *Engine) workflow(name string) (Workflow, bool) { func (e *Engine) workflow(name string) (registered, bool) {
e.mu.RLock() e.mu.RLock()
defer e.mu.RUnlock() defer e.mu.RUnlock()
wf, ok := e.workflows[name] wf, ok := e.workflows[name]
+48 -5
View File
@@ -5,6 +5,7 @@ import (
"encoding/json/jsontext" "encoding/json/jsontext"
json "encoding/json/v2" json "encoding/json/v2"
"log/slog" "log/slog"
"strings"
"sync" "sync"
"testing" "testing"
"time" "time"
@@ -28,6 +29,7 @@ type fakeWork struct {
rejectFrom *int rejectFrom *int
steps []store.Step steps []store.Step
finished []string finished []string
errors []string
results []jsontext.Value results []jsontext.Value
} }
@@ -67,13 +69,14 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st
return nil return nil
} }
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, _ string) error { func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error {
f.mu.Lock() f.mu.Lock()
defer f.mu.Unlock() defer f.mu.Unlock()
if f.leaseLost { if f.leaseLost {
return store.ErrLeaseLost return store.ErrLeaseLost
} }
f.finished = append(f.finished, state) f.finished = append(f.finished, state)
f.errors = append(f.errors, errText)
f.results = append(f.results, result) f.results = append(f.results, result)
return nil return nil
} }
@@ -130,7 +133,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{}) engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{})
effects := 0 effects := 0
engine.Register("probe", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { engine.Register("probe", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
out, err := rc.Step("probe", func(context.Context, string) (string, error) { out, err := rc.Step("probe", func(context.Context, string) (string, error) {
return "mine", nil return "mine", nil
}) })
@@ -145,7 +148,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
} }
return json.Marshal(out) return json.Marshal(out)
}) })
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Input: jsontext.Value(`null`)} run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Version: 1, Input: jsontext.Value(`null`)}
engine.execute(t.Context(), run, uuid.New()) engine.execute(t.Context(), run, uuid.New())
if got := len(tt.work.steps); got != tt.wantSteps { if got := len(tt.work.steps); got != tt.wantSteps {
@@ -175,7 +178,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
} }
var seen []string var seen []string
register := func(e *Engine) { register := func(e *Engine) {
e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { e.Register("pin", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) { got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
}) })
@@ -186,7 +189,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
return jsontext.Value(`null`), nil return jsontext.Value(`null`), nil
}) })
} }
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)} run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Version: 1, Input: jsontext.Value(`null`)}
live := &fakeWork{} live := &fakeWork{}
e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{}) e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{})
@@ -202,3 +205,43 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
t.Fatalf("live and replayed values differ: %q", seen) t.Fatalf("live and replayed values differ: %q", seen)
} }
} }
// TestRunExecutesOnlyUnderItsStartVersion guards workflow versioning: a run
// resumed after its workflow's version changed must fail naming both versions
// and run none of the new code, never replay old steps through new logic.
func TestRunExecutesOnlyUnderItsStartVersion(t *testing.T) {
tests := []struct {
name string
runVersion int
wantState string
wantRan bool
wantErrPart string
}{
{name: "same version runs", runVersion: 2, wantState: store.StateSucceeded, wantRan: true},
{name: "older run fails naming both versions", runVersion: 1, wantState: store.StateFailed, wantErrPart: "version 1"},
{name: "newer run fails when the build is behind", runVersion: 3, wantState: store.StateFailed, wantErrPart: "version 2"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
work := &fakeWork{}
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
ran := false
engine.Register("versioned", 2, func(*Context, jsontext.Value) (jsontext.Value, error) {
ran = true
return jsontext.Value(`null`), nil
})
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "versioned", Version: tt.runVersion, Input: jsontext.Value(`null`)}
engine.execute(t.Context(), run, uuid.New())
if ran != tt.wantRan {
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
}
if len(work.finished) != 1 || work.finished[0] != tt.wantState {
t.Fatalf("finished = %v, want [%s]", work.finished, tt.wantState)
}
if tt.wantErrPart != "" && !strings.Contains(work.errors[0], tt.wantErrPart) {
t.Fatalf("error %q does not name %q", work.errors[0], tt.wantErrPart)
}
})
}
}
+2 -1
View File
@@ -40,6 +40,7 @@ type Origins interface {
type RunView struct { type RunView struct {
ID uuid.UUID `json:"id"` ID uuid.UUID `json:"id"`
Workflow string `json:"workflow"` Workflow string `json:"workflow"`
Version int `json:"workflow_version"`
State string `json:"state"` State string `json:"state"`
Attempts int `json:"attempts"` Attempts int `json:"attempts"`
Error string `json:"error,omitzero"` Error string `json:"error,omitzero"`
@@ -292,7 +293,7 @@ func effectOf(s store.Recorded) *Effect {
func view(s store.Summary, result jsontext.Value) RunView { func view(s store.Summary, result jsontext.Value) RunView {
v := RunView{ v := RunView{
ID: s.ID, Workflow: s.Workflow, State: s.State, Attempts: s.Attempts, ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result, Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result,
} }
if s.FinishedAt.Valid { if s.FinishedAt.Valid {
+1
View File
@@ -44,6 +44,7 @@ export interface Usage {
export interface Run { export interface Run {
id: string; id: string;
workflow: string; workflow: string;
workflow_version: number;
state: "queued" | "running" | "waiting" | "succeeded" | "failed" | "cancelled"; state: "queued" | "running" | "waiting" | "succeeded" | "failed" | "cancelled";
attempts: number; attempts: number;
error?: string; error?: string;