diff --git a/modules/runs/corpus_test.go b/modules/runs/corpus_test.go index db04d25e9..647c77fc4 100644 --- a/modules/runs/corpus_test.go +++ b/modules/runs/corpus_test.go @@ -127,6 +127,13 @@ steps: settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}} `, }, + { + name: "a step id with a dot is refused, so a reference reads one way", + yaml: `steps: + - {id: plan.v2, type: run.notify} +`, + want: []string{`step id "plan.v2" must be lowercase letters and digits, joined by hyphens`}, + }, { name: "a duplicate step id is refused", yaml: `steps: diff --git a/modules/runs/dry.go b/modules/runs/dry.go index aefa33982..7b3ae3559 100644 --- a/modules/runs/dry.go +++ b/modules/runs/dry.go @@ -73,7 +73,13 @@ type DryRequest struct { // StartDry queues a dry run of a document and returns its run's id. A document // that doesn't validate starts nothing. func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) { - if err := e.validateDocument(ctx, req.Document); err != nil { + doc, err := e.validateDocument(ctx, req.Document) + if err != nil { + return uuid.Nil, err + } + // A dry run starts like a live one: a document that can't start can't be + // tested, and its inputs are checked the same way. + if err := doc.unsupported(); err != nil { return uuid.Nil, err } overlay, err := json.Marshal(dryOverlay{Document: req.Document, PolicyKind: req.PolicyKind, Policy: req.Policy}) @@ -84,6 +90,9 @@ func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error if len(input) == 0 { input = jsontext.Value("null") } + if err := doc.checkInputs(decodeInputs(input)); err != nil { + return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err) + } return e.db.Start(ctx, store.NewRun{ OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay, }) diff --git a/modules/runs/dry_test.go b/modules/runs/dry_test.go index db0b4c975..ca3132312 100644 --- a/modules/runs/dry_test.go +++ b/modules/runs/dry_test.go @@ -3,6 +3,8 @@ package runs import ( "context" "encoding/json/jsontext" + "log/slog" + "strings" "testing" "github.com/google/uuid" @@ -40,6 +42,45 @@ type anyBox struct{} func (anyBox) Declarations() []connectors.Declaration { return nil } func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true } +// TestStartDryRefusesWhatCannotStart guards the dry-run entry point against +// testing a document a live run could not start: a declared identity or +// profile, or a missing required input. The document is refused before a run +// is queued. +func TestStartDryRefusesWhatCannotStart(t *testing.T) { + engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{}) + tests := []struct { + name string + doc string + want string + }{ + { + name: "identity", + doc: "repository: none\nidentity: archie\nsteps:\n - {type: run.notify}\n", + want: "acts as identity", + }, + { + name: "profile", + doc: "repository: none\nprofile: build\nsteps:\n - {type: run.notify}\n", + want: "runs under profile", + }, + { + name: "required input", + doc: "inputs: {event: {type: object, required: true}}\nrepository: none\nsteps:\n - {type: run.notify}\n", + want: `input "event" is required`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := engine.StartDry(context.Background(), DryRequest{ + OrgID: "org", Workflow: "draft", Document: tt.doc, + }) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("StartDry error = %v, want %q", err, tt.want) + } + }) + } +} + // TestDryRunNeverExecutesAChangingAction guards the dry run's promise: an // action that changes something is judged and never executed, however freely // judgement would let it run, while one that only reads runs for real. diff --git a/modules/runs/interpreter.go b/modules/runs/interpreter.go index 8cc455007..516abe1b5 100644 --- a/modules/runs/interpreter.go +++ b/modules/runs/interpreter.go @@ -146,7 +146,7 @@ type progress struct { // dry run (rc.Dry) judges every action and executes only those that read. func (d Deps) Compile(doc Document) Workflow { aliases, withMemory := agentUse(doc) - settleAtEnd := heldAfterLastApproval(doc) + settleAtEnd := settlesAtEnd(doc) return func(rc *Context, input jsontext.Value) (jsontext.Value, error) { p := &progress{ input: input, inputs: decodeInputs(input), instructions: doc.Instructions, @@ -165,8 +165,8 @@ func (d Deps) Compile(doc Document) Workflow { return nil, err } } - // An action held after the last approval step is still put to people - // before the run ends, so nothing judgement held runs unapproved or is + // An action still held when the document ends is put to people before + // the run finishes, so nothing judgement held runs unapproved or is // silently dropped. if settleAtEnd && !p.stopped { if _, err := d.settle(rc, p); err != nil { @@ -241,19 +241,25 @@ func modelAlias(alias string) string { return alias } -// heldAfterLastApproval reports whether a step that can propose an action comes -// after the document's last approval step, or there is none. -func heldAfterLastApproval(doc Document) bool { - held := false +// settlesAtEnd reports whether the run must put held actions to people after +// its last step. It does when a step that can propose an action comes after the +// last approval step, or when that approval step is conditional: a skipped +// approval leaves everything held before it unsettled. +func settlesAtEnd(doc Document) bool { + settled := true for _, s := range doc.Steps { switch { + case s.Type == typeApproval && s.When == "" && s.OnFailure != onFailureContinue: + settled = true case s.Type == typeApproval: - held = false + // A conditional approval may not run, and one that continues past + // its own failure may not settle what it asked, so anything held + // before it stays unsettled unless a later one settles it. case holds(s.Type): - held = true + settled = false } } - return held + return !settled } // decodeInputs reads the run's input as the object references resolve inputs diff --git a/modules/runs/investigate_test.go b/modules/runs/investigate_test.go new file mode 100644 index 000000000..1b02b254d --- /dev/null +++ b/modules/runs/investigate_test.go @@ -0,0 +1,221 @@ +package runs + +import ( + "context" + "encoding/json/jsontext" + json "encoding/json/v2" + "errors" + "log/slog" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/samcharles93/ai-sdk/chat" + + "github.com/samcharles93/archie-core/modules/approvals" + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/judgement" + "github.com/samcharles93/archie-core/modules/memory" + "github.com/samcharles93/archie-core/modules/runs/internal/store" +) + +// TestContinuedFailureDoesNotSwallowSuspension guards the run's stop signals: +// on_failure continue covers a step's own failure, never a suspension, a lost +// lease or a cancellation. A continued suspension would run the rest of the +// workflow while the step is still waiting. +func TestContinuedFailureDoesNotSwallowSuspension(t *testing.T) { + doc, err := ParseDocument(`steps: + - {type: workflow.sleep, on_failure: continue, settings: {for: 1h}} + - {type: run.notify, settings: {title: too soon}} +`) + if err != nil { + t.Fatalf("ParseDocument: %v", err) + } + told := ¬ices{} + wf := Deps{Settings: onePersonPath{}, Judge: &actionJudge{}, Notifier: told}.Compile(doc) + rc := newContext(t.Context(), &fakeWork{}, store.Run{ID: uuid.New()}, uuid.New(), nil) + if _, err := wf(rc, jsontext.Value(`null`)); !errors.Is(err, errSuspended) { + t.Fatalf("run error = %v, want a suspension", err) + } + if len(told.sent) != 0 { + t.Fatalf("a suspended run sent %d notifications", len(told.sent)) + } +} + +// investigateModel calls the action tool once, reports findings once the tool +// has answered, and answers any later turn (such as recording the cause) +// without a tool call. +type investigateModel struct { + tool string + calls int +} + +func (m *investigateModel) Name() string { return "investigate-fake" } + +func (m *investigateModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) { + m.calls++ + for _, tool := range req.Tools { + if tool.Name != m.tool { + continue + } + last := req.Messages[len(req.Messages)-1] + if last.Role == chat.RoleTool { + return chat.Response{ + Role: chat.RoleAssistant, FinishReason: "stop", + Content: "web-1 is out of disk; restart it.", + }, nil + } + return chat.Response{ + Role: chat.RoleAssistant, FinishReason: "tool_calls", + ToolCalls: []chat.ToolCall{{ID: "call-1", Name: m.tool, Arguments: `{"target":"web-1"}`}}, + }, nil + } + return chat.Response{Role: chat.RoleAssistant, FinishReason: "stop", Content: "no incident recorded"}, nil +} + +func (m *investigateModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) { + return nil, errors.New("not streamed") +} + +// investigateSettings resolves the default agent model and offers the fake +// model to every agent step. +type investigateSettings struct { + onePersonPath + model chat.Provider +} + +func (s investigateSettings) AgentModelForAlias(context.Context, string) (governance.Model, error) { + return governance.Model{Model: "test"}, nil +} + +func (s investigateSettings) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) { + return s.model, chat.ModelInfo{}, nil +} + +func (investigateSettings) AttemptsPerRun(context.Context) (int, error) { return 8, nil } + +// EscalationPath waits long enough that the first attempt suspends, as a +// person who has not answered yet does. +func (investigateSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) { + return []governance.PathStep{{Person: "p", DeadlineMS: time.Hour.Milliseconds()}}, nil +} + +// investigateBox declares one action so the model has something to call. The +// judge decides it, so the connector itself is never invoked. +type investigateBox struct{} + +func (investigateBox) Declarations() []connectors.Declaration { + return []connectors.Declaration{{ + Connector: "host", Kind: connectors.KindAction, Name: "restart", + Description: "Restart a host", + Schema: jsontext.Value(`{"type":"object","properties":{"target":{"type":"string"}},"required":["target"]}`), + }} +} + +func (investigateBox) Tool(string) (connectors.Tool, bool) { return nil, true } + +// heldJudge is actionJudge with the real Execute contract for held actions: a +// held action is not run by the agent's tool call, only by an approval. +type heldJudge struct{ actionJudge } + +func (j *heldJudge) Execute(ctx context.Context, d judgement.Decision, input jsontext.Value, key string) (string, error) { + if !d.Level.Runs() { + return "", judgement.ErrHeld + } + return j.actionJudge.Execute(ctx, d, input, key) +} + +// investigateMemory records nothing; the workflow only needs the finish step. +type investigateMemory struct{} + +func (investigateMemory) Past(context.Context, uuid.UUID, memory.Target) ([]memory.Incident, error) { + return nil, nil +} + +func (investigateMemory) Record(context.Context, uuid.UUID, jsontext.Value, memory.Target, string, string) error { + return nil +} + +func (investigateMemory) Finish(context.Context, uuid.UUID, []memory.Action, int) (memory.Review, error) { + return memory.Review{}, nil +} + +// TestInvestigateDocumentSuspendsAndResumes guards the document the binary +// ships: it investigates the event, holds its action for approval, suspends +// while the person decides, and on resume replays the agent and the request +// instead of asking or running anything twice. +func TestInvestigateDocumentSuspendsAndResumes(t *testing.T) { + doc, err := ParseDocument(InvestigateDocument()) + if err != nil { + t.Fatalf("the shipped investigate document doesn't parse: %v", err) + } + model := &investigateModel{tool: toolName("host", "restart")} + judge := &heldJudge{actionJudge{levels: map[string]judgement.Level{"restart": judgement.ProposeAndWait}}} + asked := &askCounter{} + told := ¬ices{} + deps := Deps{ + Settings: investigateSettings{model: model}, Toolbox: investigateBox{}, + Judge: judge, Approver: asked, Notifier: told, Memory: investigateMemory{}, + } + run := store.Run{ + ID: uuid.New(), OrgID: "org", Workflow: "investigate", Version: 1, + Input: jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`), + } + + // The first attempt proposes the action, asks once and suspends on the + // person's answer. + first := &fakeWork{} + engine := newEngine(nil, first, slog.New(slog.DiscardHandler), Options{}) + useWorkflow(engine, deps.Compile(doc)) + engine.execute(t.Context(), run, uuid.New()) + if len(first.finished) != 0 { + t.Fatalf("the first attempt finished %v instead of suspending", first.finished) + } + if asked.asked != 1 || judge.executed != 0 { + t.Fatalf("first attempt asked %d and executed %d, want 1 and 0", asked.asked, judge.executed) + } + + // The resumed attempt replays the agent and the request, takes the answer, + // runs the action once and reports one notification. + calls := model.calls + resumed := &signalAfter{steps: first.steps, arrives: 1} + engine = newEngine(nil, resumed, slog.New(slog.DiscardHandler), Options{}) + useWorkflow(engine, deps.Compile(doc)) + engine.execute(t.Context(), run, uuid.New()) + if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded { + t.Fatalf("resumed run finished %v, want one success", resumed.finished) + } + if asked.asked != 1 { + t.Fatalf("the resumed run asked %d times in total, want the first attempt's one", asked.asked) + } + if judge.executed != 1 { + t.Fatalf("the resumed run executed the action %d times, want once", judge.executed) + } + if model.calls != calls { + t.Fatalf("the resumed run called the model %d times, want none", model.calls-calls) + } + if len(told.sent) != 1 { + t.Fatalf("the run sent %d notifications, want one", len(told.sent)) + } + if body := told.sent[0].Body; !strings.Contains(body, "restart") || !strings.Contains(body, approvals.StateApproved) { + t.Fatalf("the notification doesn't report the approved action: %q", body) + } + asks := 0 + for _, s := range resumed.steps { + if strings.HasPrefix(s.Name, "ask:") { + asks++ + } + } + if asks != 1 { + t.Fatalf("the run recorded %d approval requests, want one", asks) + } + var result AgentResult + if err := json.Unmarshal(resumed.results[0], &result); err != nil { + t.Fatalf("run result: %v", err) + } + if result.Answer == "" { + t.Fatalf("run result has no findings: %s", resumed.results[0]) + } +} diff --git a/modules/runs/references.go b/modules/runs/references.go index c121ab2c5..b843ec7b4 100644 --- a/modules/runs/references.go +++ b/modules/runs/references.go @@ -7,8 +7,6 @@ import ( "strings" "gopkg.in/yaml.v3" - - "github.com/samcharles93/archie-core/internal/platform/stableid" ) // StepResult is the output later steps can reference as @@ -52,14 +50,18 @@ func (s *refScope) has(id string) bool { return ok } -// checkStepID requires a declared step id to be a stable identifier and -// unique within the workflow. +// stepIDPattern is narrower than a stable identifier: a step id is read inside +// a dotted reference, so a dot in the id would make the reference ambiguous. +var stepIDPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$`) + +// checkStepID requires a declared step id to be unique within the workflow and +// readable inside a dotted reference. func checkStepID(id string, seen *refScope) error { if id == "" { return nil } - if !stableid.Valid(id) { - return fmt.Errorf("step id %q is not a stable identifier: lowercase letters and digits, joined by dots or hyphens", id) + if !stepIDPattern.MatchString(id) { + return fmt.Errorf("step id %q must be lowercase letters and digits, joined by hyphens", id) } if seen.has(id) { return fmt.Errorf("step id %q is declared twice", id) diff --git a/modules/runs/runs.go b/modules/runs/runs.go index d7642323a..a77453b35 100644 --- a/modules/runs/runs.go +++ b/modules/runs/runs.go @@ -108,7 +108,7 @@ func (e *Engine) steps() (Deps, error) { // what the latest version already is, and returns the version in force. A body // that doesn't validate is stored as nothing. func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int, error) { - if err := e.validateDocument(ctx, body); err != nil { + if _, err := e.validateDocument(ctx, body); err != nil { return 0, err } latest, err := e.db.Workflow(ctx, orgID, name, 0) @@ -125,10 +125,10 @@ func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int return w.Version, nil } -func (e *Engine) validateDocument(ctx context.Context, body string) error { +func (e *Engine) validateDocument(ctx context.Context, body string) (Document, error) { doc, err := ParseDocument(body) if err != nil { - return err + return Document{}, err } var problems []Problem for i, step := range doc.Steps { @@ -137,26 +137,26 @@ func (e *Engine) validateDocument(ctx context.Context, body string) error { } var s agentSettings if err := decodeSettings(step.Settings, &s); err != nil { - return err + return Document{}, err } if modelAlias(s.Model) == "" { continue } deps, err := e.steps() if err != nil { - return err + return Document{}, err } if _, err := deps.Settings.AgentModelForAlias(ctx, s.Model); err != nil { if !errors.Is(err, governance.ErrNoAgentModel) { - return err + return Document{}, err } problems = append(problems, Problem{Path: fmt.Sprintf("steps[%d].settings.model", i), Message: fmt.Sprintf("unknown agent model alias %q", s.Model)}) } } if len(problems) > 0 { - return &InvalidError{Problems: problems} + return Document{}, &InvalidError{Problems: problems} } - return nil + return doc, nil } // Has reports whether a workflow is stored under name. @@ -215,6 +215,9 @@ func (e *Engine) CheckTrigger(ctx context.Context, orgID, workflow string) error if err != nil { return refuse("workflow %q is not valid: %v", workflow, err) } + if err := doc.unsupported(); err != nil { + return refuse("workflow %q can't start: %v", workflow, err) + } if missing := doc.requires("event"); len(missing) > 0 { return refuse("workflow %q requires %s, and a trigger supplies only event", workflow, strings.Join(missing, ", ")) } diff --git a/modules/runs/steps_test.go b/modules/runs/steps_test.go index a73dcce24..9c57cd06d 100644 --- a/modules/runs/steps_test.go +++ b/modules/runs/steps_test.go @@ -53,11 +53,19 @@ func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, return j.decisions, nil } -// askCounter approves every request at once and counts them. -type askCounter struct{ asked int } +// askCounter approves every request at once and counts them. failFirst makes +// that many requests fail first, as an approvals store that is briefly down. +type askCounter struct { + asked int + failFirst int +} func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) { a.asked++ + if a.failFirst > 0 { + a.failFirst-- + return approvals.Request{}, errors.New("approvals unavailable") + } return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil } @@ -106,6 +114,7 @@ func TestDocumentStepsRun(t *testing.T) { doc string levels map[string]judgement.Level fail error + failFirst int wantErr bool wantInput string wantAssessed int @@ -153,6 +162,26 @@ steps: levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait}, wantAssessed: 2, wantAsked: 1, wantExecuted: 1, }, + { + name: "a held action is settled when the last approval step is skipped", + doc: `inputs: {skip: {type: bool}} +steps: + - {type: connector.action, settings: {connector: host, name: clean}} + - {type: approval.wait, when: "inputs.skip"} +`, + levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait}, + wantAssessed: 2, wantAsked: 1, wantExecuted: 1, + }, + { + name: "an approval that fails and continues is settled before the run ends", + doc: `steps: + - {type: connector.action, settings: {connector: host, name: clean}} + - {type: approval.wait, on_failure: continue} +`, + levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait}, + failFirst: 1, + wantAssessed: 2, wantAsked: 2, wantExecuted: 1, + }, { name: "a later approval step asks only about what was held since the last", doc: `steps: @@ -181,7 +210,7 @@ steps: t.Fatalf("ParseDocument: %v", err) } judge := &actionJudge{levels: tt.levels, fail: tt.fail} - ask := &askCounter{} + ask := &askCounter{failFirst: tt.failFirst} told := ¬ices{} wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc) work := &signalAfter{arrives: 1}