feat(runs): a run is pinned to the workflow version it started with and fails clearly when it changed (archie #2014)
This commit is contained in:
8 files changed
+99
-28
No files matched your search
@@ -94,7 +94,7 @@ func Run(ctx context.Context, configPath string) error {
|
||||
allow.Routes(mux, guard)
|
||||
classes.Routes(mux, guard, allow)
|
||||
remembered := memory.NewModule(db, orgID)
|
||||
engine.Register("investigate", runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above
|
||||
engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above
|
||||
|
||||
// An answer a channel delivers is an event, and settles its request.
|
||||
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
-- +goose Up
|
||||
-- Runs that exist before versions were declared started under version 1.
|
||||
ALTER TABLE runs.runs ADD COLUMN workflow_version integer NOT NULL DEFAULT 1;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE runs.runs DROP COLUMN workflow_version;
|
||||
@@ -118,7 +118,7 @@ func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
|
||||
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
|
||||
t.Helper()
|
||||
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
||||
engine.Register("agent", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
engine.Register("agent", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -141,7 +141,7 @@ func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provi
|
||||
}
|
||||
return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) })
|
||||
})
|
||||
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Input: jsontext.Value(`null`)}
|
||||
run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)}
|
||||
engine.execute(ctx, run, uuid.New())
|
||||
}
|
||||
|
||||
|
||||
@@ -41,9 +41,11 @@ var (
|
||||
|
||||
// Run is a claimed run.
|
||||
type Run struct {
|
||||
ID uuid.UUID `db:"id"`
|
||||
OrgID string `db:"org_id"`
|
||||
Workflow string `db:"workflow"`
|
||||
ID uuid.UUID `db:"id"`
|
||||
OrgID string `db:"org_id"`
|
||||
Workflow string `db:"workflow"`
|
||||
// Version is the workflow version the run started under.
|
||||
Version int `db:"workflow_version"`
|
||||
Input jsontext.Value `db:"input"`
|
||||
Attempts int `db:"attempts"`
|
||||
}
|
||||
@@ -70,14 +72,14 @@ func New(db *sqlx.DB) *Store {
|
||||
|
||||
// Start inserts a queued run. A run with the same organisation and start key
|
||||
// is not inserted again; its id is returned instead.
|
||||
func (s *Store) Start(ctx context.Context, orgID, workflow, startKey string, input jsontext.Value) (uuid.UUID, error) {
|
||||
func (s *Store) Start(ctx context.Context, orgID, workflow string, version int, startKey string, input jsontext.Value) (uuid.UUID, error) {
|
||||
key := sql.NullString{String: startKey, Valid: startKey != ""}
|
||||
var id uuid.UUID
|
||||
err := s.db.GetContext(ctx, &id, `
|
||||
INSERT INTO runs.runs (id, org_id, workflow, start_key, input, state)
|
||||
VALUES ($1, $2, $3, $4, $5, 'queued')
|
||||
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'queued')
|
||||
ON CONFLICT (org_id, start_key) DO NOTHING
|
||||
RETURNING id`, uuid.New(), orgID, workflow, key, input)
|
||||
RETURNING id`, uuid.New(), orgID, workflow, version, key, input)
|
||||
if errors.Is(err, sql.ErrNoRows) && key.Valid {
|
||||
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey)
|
||||
}
|
||||
@@ -105,7 +107,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti
|
||||
ORDER BY created_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1)
|
||||
RETURNING r.id, r.org_id, r.workflow, r.input, r.attempts`, token, owner, ttl.Seconds())
|
||||
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts`, token, owner, ttl.Seconds())
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Run{}, false, nil
|
||||
}
|
||||
@@ -253,6 +255,7 @@ func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error)
|
||||
type Summary struct {
|
||||
ID uuid.UUID `db:"id"`
|
||||
Workflow string `db:"workflow"`
|
||||
Version int `db:"workflow_version"`
|
||||
State string `db:"state"`
|
||||
Attempts int `db:"attempts"`
|
||||
Error sql.NullString `db:"error"`
|
||||
@@ -279,7 +282,7 @@ type Recorded struct {
|
||||
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
|
||||
var out []Summary
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT id, workflow, state, attempts, error, created_at, finished_at FROM runs.runs
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at FROM runs.runs
|
||||
WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
||||
return nil, fmt.Errorf("list runs: %w", err)
|
||||
}
|
||||
@@ -291,7 +294,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
|
||||
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
|
||||
var d Detail
|
||||
err := s.db.GetContext(ctx, &d, `
|
||||
SELECT id, workflow, state, attempts, error, created_at, finished_at, input,
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, input,
|
||||
coalesce(result, 'null') AS result
|
||||
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
|
||||
+26
-9
@@ -24,6 +24,12 @@ import (
|
||||
// lookups belong inside steps.
|
||||
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
|
||||
|
||||
// registered is a workflow with the version its code declares.
|
||||
type registered struct {
|
||||
version int
|
||||
run Workflow
|
||||
}
|
||||
|
||||
// StartRequest names the workflow to run and the organisation it runs for.
|
||||
// A non-empty Key makes the start idempotent within the organisation.
|
||||
type StartRequest struct {
|
||||
@@ -60,7 +66,7 @@ type Engine struct {
|
||||
log *slog.Logger
|
||||
opts Options
|
||||
mu sync.RWMutex
|
||||
workflows map[string]Workflow
|
||||
workflows map[string]registered
|
||||
}
|
||||
|
||||
// New returns an engine on db.
|
||||
@@ -82,23 +88,30 @@ func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options
|
||||
if opts.Owner == "" {
|
||||
opts.Owner = "archie"
|
||||
}
|
||||
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]Workflow{}}
|
||||
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]registered{}}
|
||||
}
|
||||
|
||||
// Register makes a workflow available to runs by name.
|
||||
func (e *Engine) Register(name string, wf Workflow) {
|
||||
// Register makes a workflow available to runs by name. Raise version whenever
|
||||
// a change to wf would alter the steps an in-flight run replays: a run only
|
||||
// executes under the version it started with.
|
||||
func (e *Engine) Register(name string, version int, wf Workflow) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.workflows[name] = wf
|
||||
e.workflows[name] = registered{version: version, run: wf}
|
||||
}
|
||||
|
||||
// Start queues a run and returns its id.
|
||||
// Start queues a run under the registered version of its workflow and returns
|
||||
// its id.
|
||||
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
|
||||
wf, ok := e.workflow(req.Workflow)
|
||||
if !ok {
|
||||
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
|
||||
}
|
||||
input := req.Input
|
||||
if len(input) == 0 {
|
||||
input = jsontext.Value("null")
|
||||
}
|
||||
return e.db.Start(ctx, req.OrgID, req.Workflow, req.Key, input)
|
||||
return e.db.Start(ctx, req.OrgID, req.Workflow, wf.version, req.Key, input)
|
||||
}
|
||||
|
||||
// Signal delivers a payload to a run waiting on key, now or later.
|
||||
@@ -188,14 +201,18 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no workflow named %q", run.Workflow)
|
||||
}
|
||||
if run.Version != wf.version {
|
||||
return nil, fmt.Errorf("run started under version %d of workflow %q, this build runs version %d; start a new run",
|
||||
run.Version, run.Workflow, wf.version)
|
||||
}
|
||||
steps, err := e.work.Steps(ctx, run.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
|
||||
return wf.run(newContext(ctx, e.work, run, token, steps), run.Input)
|
||||
}
|
||||
|
||||
func (e *Engine) workflow(name string) (Workflow, bool) {
|
||||
func (e *Engine) workflow(name string) (registered, bool) {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
wf, ok := e.workflows[name]
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -28,6 +29,7 @@ type fakeWork struct {
|
||||
rejectFrom *int
|
||||
steps []store.Step
|
||||
finished []string
|
||||
errors []string
|
||||
results []jsontext.Value
|
||||
}
|
||||
|
||||
@@ -67,13 +69,14 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, _ string) error {
|
||||
func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseLost {
|
||||
return store.ErrLeaseLost
|
||||
}
|
||||
f.finished = append(f.finished, state)
|
||||
f.errors = append(f.errors, errText)
|
||||
f.results = append(f.results, result)
|
||||
return nil
|
||||
}
|
||||
@@ -130,7 +133,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{})
|
||||
effects := 0
|
||||
engine.Register("probe", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
engine.Register("probe", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
out, err := rc.Step("probe", func(context.Context, string) (string, error) {
|
||||
return "mine", nil
|
||||
})
|
||||
@@ -145,7 +148,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
|
||||
}
|
||||
return json.Marshal(out)
|
||||
})
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Input: jsontext.Value(`null`)}
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Version: 1, Input: jsontext.Value(`null`)}
|
||||
engine.execute(t.Context(), run, uuid.New())
|
||||
|
||||
if got := len(tt.work.steps); got != tt.wantSteps {
|
||||
@@ -175,7 +178,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
||||
}
|
||||
var seen []string
|
||||
register := func(e *Engine) {
|
||||
e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
e.Register("pin", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
|
||||
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
|
||||
})
|
||||
@@ -186,7 +189,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
||||
return jsontext.Value(`null`), nil
|
||||
})
|
||||
}
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)}
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Version: 1, Input: jsontext.Value(`null`)}
|
||||
|
||||
live := &fakeWork{}
|
||||
e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{})
|
||||
@@ -202,3 +205,43 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
||||
t.Fatalf("live and replayed values differ: %q", seen)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunExecutesOnlyUnderItsStartVersion guards workflow versioning: a run
|
||||
// resumed after its workflow's version changed must fail naming both versions
|
||||
// and run none of the new code, never replay old steps through new logic.
|
||||
func TestRunExecutesOnlyUnderItsStartVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
runVersion int
|
||||
wantState string
|
||||
wantRan bool
|
||||
wantErrPart string
|
||||
}{
|
||||
{name: "same version runs", runVersion: 2, wantState: store.StateSucceeded, wantRan: true},
|
||||
{name: "older run fails naming both versions", runVersion: 1, wantState: store.StateFailed, wantErrPart: "version 1"},
|
||||
{name: "newer run fails when the build is behind", runVersion: 3, wantState: store.StateFailed, wantErrPart: "version 2"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
work := &fakeWork{}
|
||||
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
||||
ran := false
|
||||
engine.Register("versioned", 2, func(*Context, jsontext.Value) (jsontext.Value, error) {
|
||||
ran = true
|
||||
return jsontext.Value(`null`), nil
|
||||
})
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "versioned", Version: tt.runVersion, Input: jsontext.Value(`null`)}
|
||||
engine.execute(t.Context(), run, uuid.New())
|
||||
|
||||
if ran != tt.wantRan {
|
||||
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
|
||||
}
|
||||
if len(work.finished) != 1 || work.finished[0] != tt.wantState {
|
||||
t.Fatalf("finished = %v, want [%s]", work.finished, tt.wantState)
|
||||
}
|
||||
if tt.wantErrPart != "" && !strings.Contains(work.errors[0], tt.wantErrPart) {
|
||||
t.Fatalf("error %q does not name %q", work.errors[0], tt.wantErrPart)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ type Origins interface {
|
||||
type RunView struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
Workflow string `json:"workflow"`
|
||||
Version int `json:"workflow_version"`
|
||||
State string `json:"state"`
|
||||
Attempts int `json:"attempts"`
|
||||
Error string `json:"error,omitzero"`
|
||||
@@ -292,7 +293,7 @@ func effectOf(s store.Recorded) *Effect {
|
||||
|
||||
func view(s store.Summary, result jsontext.Value) RunView {
|
||||
v := RunView{
|
||||
ID: s.ID, Workflow: s.Workflow, State: s.State, Attempts: s.Attempts,
|
||||
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
|
||||
Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result,
|
||||
}
|
||||
if s.FinishedAt.Valid {
|
||||
|
||||
@@ -44,6 +44,7 @@ export interface Usage {
|
||||
export interface Run {
|
||||
id: string;
|
||||
workflow: string;
|
||||
workflow_version: number;
|
||||
state: "queued" | "running" | "waiting" | "succeeded" | "failed" | "cancelled";
|
||||
attempts: number;
|
||||
error?: string;
|
||||
|
||||
Reference in new issue
Block a user