diff --git a/internal/app/archie/run.go b/internal/app/archie/run.go index b9922a3bf..ac9501bf1 100644 --- a/internal/app/archie/run.go +++ b/internal/app/archie/run.go @@ -94,7 +94,7 @@ func Run(ctx context.Context, configPath string) error { allow.Routes(mux, guard) classes.Routes(mux, guard, allow) remembered := memory.NewModule(db, orgID) - engine.Register("investigate", runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above + engine.Register("investigate", 1, runs.Investigate(settings, conns, judge, allow, tell, remembered)) //nolint:contextcheck // see above // An answer a channel delivers is an event, and settles its request. hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow) diff --git a/internal/platform/pg/migrations/00023_runs_workflow_version.sql b/internal/platform/pg/migrations/00023_runs_workflow_version.sql new file mode 100644 index 000000000..c636967a3 --- /dev/null +++ b/internal/platform/pg/migrations/00023_runs_workflow_version.sql @@ -0,0 +1,6 @@ +-- +goose Up +-- Runs that exist before versions were declared started under version 1. +ALTER TABLE runs.runs ADD COLUMN workflow_version integer NOT NULL DEFAULT 1; + +-- +goose Down +ALTER TABLE runs.runs DROP COLUMN workflow_version; diff --git a/modules/runs/agent_test.go b/modules/runs/agent_test.go index 05f9def58..402eb3b15 100644 --- a/modules/runs/agent_test.go +++ b/modules/runs/agent_test.go @@ -118,7 +118,7 @@ func TestAgentResumesFromItsRecordedCalls(t *testing.T) { func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) { t.Helper() engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{}) - engine.Register("agent", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { + engine.Register("agent", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil { return nil, err } @@ -141,7 +141,7 @@ func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provi } return rc.Step("after", func(context.Context, string) (jsontext.Value, error) { return json.Marshal(res) }) }) - run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Input: jsontext.Value(`null`)} + run := store.Run{ID: runID, OrgID: "org", Workflow: "agent", Version: 1, Input: jsontext.Value(`null`)} engine.execute(ctx, run, uuid.New()) } diff --git a/modules/runs/internal/store/store.go b/modules/runs/internal/store/store.go index caabd3731..77c403a14 100644 --- a/modules/runs/internal/store/store.go +++ b/modules/runs/internal/store/store.go @@ -41,9 +41,11 @@ var ( // Run is a claimed run. type Run struct { - ID uuid.UUID `db:"id"` - OrgID string `db:"org_id"` - Workflow string `db:"workflow"` + ID uuid.UUID `db:"id"` + OrgID string `db:"org_id"` + Workflow string `db:"workflow"` + // Version is the workflow version the run started under. + Version int `db:"workflow_version"` Input jsontext.Value `db:"input"` Attempts int `db:"attempts"` } @@ -70,14 +72,14 @@ func New(db *sqlx.DB) *Store { // Start inserts a queued run. A run with the same organisation and start key // is not inserted again; its id is returned instead. -func (s *Store) Start(ctx context.Context, orgID, workflow, startKey string, input jsontext.Value) (uuid.UUID, error) { +func (s *Store) Start(ctx context.Context, orgID, workflow string, version int, startKey string, input jsontext.Value) (uuid.UUID, error) { key := sql.NullString{String: startKey, Valid: startKey != ""} var id uuid.UUID err := s.db.GetContext(ctx, &id, ` - INSERT INTO runs.runs (id, org_id, workflow, start_key, input, state) - VALUES ($1, $2, $3, $4, $5, 'queued') + INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state) + VALUES ($1, $2, $3, $4, $5, $6, 'queued') ON CONFLICT (org_id, start_key) DO NOTHING - RETURNING id`, uuid.New(), orgID, workflow, key, input) + RETURNING id`, uuid.New(), orgID, workflow, version, key, input) if errors.Is(err, sql.ErrNoRows) && key.Valid { err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey) } @@ -105,7 +107,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) - RETURNING r.id, r.org_id, r.workflow, r.input, r.attempts`, token, owner, ttl.Seconds()) + RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts`, token, owner, ttl.Seconds()) if errors.Is(err, sql.ErrNoRows) { return Run{}, false, nil } @@ -253,6 +255,7 @@ func (s *Store) CancelRequested(ctx context.Context, id uuid.UUID) (bool, error) type Summary struct { ID uuid.UUID `db:"id"` Workflow string `db:"workflow"` + Version int `db:"workflow_version"` State string `db:"state"` Attempts int `db:"attempts"` Error sql.NullString `db:"error"` @@ -279,7 +282,7 @@ type Recorded struct { func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) { var out []Summary if err := s.db.SelectContext(ctx, &out, ` - SELECT id, workflow, state, attempts, error, created_at, finished_at FROM runs.runs + SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at FROM runs.runs WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil { return nil, fmt.Errorf("list runs: %w", err) } @@ -291,7 +294,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) { var d Detail err := s.db.GetContext(ctx, &d, ` - SELECT id, workflow, state, attempts, error, created_at, finished_at, input, + SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, input, coalesce(result, 'null') AS result FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id) if errors.Is(err, sql.ErrNoRows) { diff --git a/modules/runs/runs.go b/modules/runs/runs.go index 24ef430f1..775c95e79 100644 --- a/modules/runs/runs.go +++ b/modules/runs/runs.go @@ -24,6 +24,12 @@ import ( // lookups belong inside steps. type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error) +// registered is a workflow with the version its code declares. +type registered struct { + version int + run Workflow +} + // StartRequest names the workflow to run and the organisation it runs for. // A non-empty Key makes the start idempotent within the organisation. type StartRequest struct { @@ -60,7 +66,7 @@ type Engine struct { log *slog.Logger opts Options mu sync.RWMutex - workflows map[string]Workflow + workflows map[string]registered } // New returns an engine on db. @@ -82,23 +88,30 @@ func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options if opts.Owner == "" { opts.Owner = "archie" } - return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]Workflow{}} + return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]registered{}} } -// Register makes a workflow available to runs by name. -func (e *Engine) Register(name string, wf Workflow) { +// Register makes a workflow available to runs by name. Raise version whenever +// a change to wf would alter the steps an in-flight run replays: a run only +// executes under the version it started with. +func (e *Engine) Register(name string, version int, wf Workflow) { e.mu.Lock() defer e.mu.Unlock() - e.workflows[name] = wf + e.workflows[name] = registered{version: version, run: wf} } -// Start queues a run and returns its id. +// Start queues a run under the registered version of its workflow and returns +// its id. func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) { + wf, ok := e.workflow(req.Workflow) + if !ok { + return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow) + } input := req.Input if len(input) == 0 { input = jsontext.Value("null") } - return e.db.Start(ctx, req.OrgID, req.Workflow, req.Key, input) + return e.db.Start(ctx, req.OrgID, req.Workflow, wf.version, req.Key, input) } // Signal delivers a payload to a run waiting on key, now or later. @@ -188,14 +201,18 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re if !ok { return nil, fmt.Errorf("no workflow named %q", run.Workflow) } + if run.Version != wf.version { + return nil, fmt.Errorf("run started under version %d of workflow %q, this build runs version %d; start a new run", + run.Version, run.Workflow, wf.version) + } steps, err := e.work.Steps(ctx, run.ID) if err != nil { return nil, err } - return wf(newContext(ctx, e.work, run, token, steps), run.Input) + return wf.run(newContext(ctx, e.work, run, token, steps), run.Input) } -func (e *Engine) workflow(name string) (Workflow, bool) { +func (e *Engine) workflow(name string) (registered, bool) { e.mu.RLock() defer e.mu.RUnlock() wf, ok := e.workflows[name] diff --git a/modules/runs/runs_test.go b/modules/runs/runs_test.go index 2bc93486d..12a739db0 100644 --- a/modules/runs/runs_test.go +++ b/modules/runs/runs_test.go @@ -5,6 +5,7 @@ import ( "encoding/json/jsontext" json "encoding/json/v2" "log/slog" + "strings" "sync" "testing" "time" @@ -28,6 +29,7 @@ type fakeWork struct { rejectFrom *int steps []store.Step finished []string + errors []string results []jsontext.Value } @@ -67,13 +69,14 @@ func (f *fakeWork) Suspend(context.Context, uuid.UUID, uuid.UUID, *time.Time, st return nil } -func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, _ string) error { +func (f *fakeWork) Finish(_ context.Context, _, _ uuid.UUID, state string, result jsontext.Value, errText string) error { f.mu.Lock() defer f.mu.Unlock() if f.leaseLost { return store.ErrLeaseLost } f.finished = append(f.finished, state) + f.errors = append(f.errors, errText) f.results = append(f.results, result) return nil } @@ -130,7 +133,7 @@ func TestLeaseFencesStepWrites(t *testing.T) { t.Run(tt.name, func(t *testing.T) { engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{}) effects := 0 - engine.Register("probe", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { + engine.Register("probe", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { out, err := rc.Step("probe", func(context.Context, string) (string, error) { return "mine", nil }) @@ -145,7 +148,7 @@ func TestLeaseFencesStepWrites(t *testing.T) { } return json.Marshal(out) }) - run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Input: jsontext.Value(`null`)} + run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "probe", Version: 1, Input: jsontext.Value(`null`)} engine.execute(t.Context(), run, uuid.New()) if got := len(tt.work.steps); got != tt.wantSteps { @@ -175,7 +178,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) { } var seen []string register := func(e *Engine) { - e.Register("pin", func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { + e.Register("pin", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) { got, err := rc.Step("pin", func(context.Context, string) (pinned, error) { return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil }) @@ -186,7 +189,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) { return jsontext.Value(`null`), nil }) } - run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Input: jsontext.Value(`null`)} + run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "pin", Version: 1, Input: jsontext.Value(`null`)} live := &fakeWork{} e := newEngine(nil, live, slog.New(slog.DiscardHandler), Options{}) @@ -202,3 +205,43 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) { t.Fatalf("live and replayed values differ: %q", seen) } } + +// TestRunExecutesOnlyUnderItsStartVersion guards workflow versioning: a run +// resumed after its workflow's version changed must fail naming both versions +// and run none of the new code, never replay old steps through new logic. +func TestRunExecutesOnlyUnderItsStartVersion(t *testing.T) { + tests := []struct { + name string + runVersion int + wantState string + wantRan bool + wantErrPart string + }{ + {name: "same version runs", runVersion: 2, wantState: store.StateSucceeded, wantRan: true}, + {name: "older run fails naming both versions", runVersion: 1, wantState: store.StateFailed, wantErrPart: "version 1"}, + {name: "newer run fails when the build is behind", runVersion: 3, wantState: store.StateFailed, wantErrPart: "version 2"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + work := &fakeWork{} + engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{}) + ran := false + engine.Register("versioned", 2, func(*Context, jsontext.Value) (jsontext.Value, error) { + ran = true + return jsontext.Value(`null`), nil + }) + run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "versioned", Version: tt.runVersion, Input: jsontext.Value(`null`)} + engine.execute(t.Context(), run, uuid.New()) + + if ran != tt.wantRan { + t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan) + } + if len(work.finished) != 1 || work.finished[0] != tt.wantState { + t.Fatalf("finished = %v, want [%s]", work.finished, tt.wantState) + } + if tt.wantErrPart != "" && !strings.Contains(work.errors[0], tt.wantErrPart) { + t.Fatalf("error %q does not name %q", work.errors[0], tt.wantErrPart) + } + }) + } +} diff --git a/modules/runs/trail.go b/modules/runs/trail.go index c8c9d7e39..b5292e265 100644 --- a/modules/runs/trail.go +++ b/modules/runs/trail.go @@ -40,6 +40,7 @@ type Origins interface { type RunView struct { ID uuid.UUID `json:"id"` Workflow string `json:"workflow"` + Version int `json:"workflow_version"` State string `json:"state"` Attempts int `json:"attempts"` Error string `json:"error,omitzero"` @@ -292,7 +293,7 @@ func effectOf(s store.Recorded) *Effect { func view(s store.Summary, result jsontext.Value) RunView { v := RunView{ - ID: s.ID, Workflow: s.Workflow, State: s.State, Attempts: s.Attempts, + ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts, Error: s.Error.String, CreatedAt: s.CreatedAt, Result: result, } if s.FinishedAt.Valid { diff --git a/ui/src/archie/api.ts b/ui/src/archie/api.ts index 7db80827c..a1cbe12c7 100644 --- a/ui/src/archie/api.ts +++ b/ui/src/archie/api.ts @@ -44,6 +44,7 @@ export interface Usage { export interface Run { id: string; workflow: string; + workflow_version: number; state: "queued" | "running" | "waiting" | "succeeded" | "failed" | "cancelled"; attempts: number; error?: string;