fix(runs): settle held actions a skipped or failed approval would leave

The end-of-run decision read the document's last approval step alone, so a conditional approval that was skipped, or one that continued past a failure, let a run finish without ever putting its held actions to people. Those approvals no longer count as settling. Resume still replays the agent and the request exactly once.

Triggers and dry runs now refuse a document that names an identity or profile, dry runs check required inputs, and step ids cannot contain dots that references read as separators.

The shipped investigate document gains a test that holds an action, suspends, resumes through the journal and reports one approved action.

Refs #2039
This commit is contained in:
sam committed 2026-11-21 01:20:21 +11:00
1 parent 0346bfef77
commit 90637aa457
8 files changed
+346 -28

No files matched your search

+7
View File
@@ -127,6 +127,13 @@ steps:
settings: {connector: host, name: clean, input: {host: "{{ inputs.event.fields.host }}"}}
`,
},
{
name: "a step id with a dot is refused, so a reference reads one way",
yaml: `steps:
- {id: plan.v2, type: run.notify}
`,
want: []string{`step id "plan.v2" must be lowercase letters and digits, joined by hyphens`},
},
{
name: "a duplicate step id is refused",
yaml: `steps:
+10 -1
View File
@@ -73,7 +73,13 @@ type DryRequest struct {
// StartDry queues a dry run of a document and returns its run's id. A document
// that doesn't validate starts nothing.
func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) {
if err := e.validateDocument(ctx, req.Document); err != nil {
doc, err := e.validateDocument(ctx, req.Document)
if err != nil {
return uuid.Nil, err
}
// A dry run starts like a live one: a document that can't start can't be
// tested, and its inputs are checked the same way.
if err := doc.unsupported(); err != nil {
return uuid.Nil, err
}
overlay, err := json.Marshal(dryOverlay{Document: req.Document, PolicyKind: req.PolicyKind, Policy: req.Policy})
@@ -84,6 +90,9 @@ func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error
if len(input) == 0 {
input = jsontext.Value("null")
}
if err := doc.checkInputs(decodeInputs(input)); err != nil {
return uuid.Nil, fmt.Errorf("workflow %q: %w", req.Workflow, err)
}
return e.db.Start(ctx, store.NewRun{
OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay,
})
+41
View File
@@ -3,6 +3,8 @@ package runs
import (
"context"
"encoding/json/jsontext"
"log/slog"
"strings"
"testing"
"github.com/google/uuid"
@@ -40,6 +42,45 @@ type anyBox struct{}
func (anyBox) Declarations() []connectors.Declaration { return nil }
func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true }
// TestStartDryRefusesWhatCannotStart guards the dry-run entry point against
// testing a document a live run could not start: a declared identity or
// profile, or a missing required input. The document is refused before a run
// is queued.
func TestStartDryRefusesWhatCannotStart(t *testing.T) {
engine := newEngine(nil, &fakeWork{}, slog.New(slog.DiscardHandler), Options{})
tests := []struct {
name string
doc string
want string
}{
{
name: "identity",
doc: "repository: none\nidentity: archie\nsteps:\n - {type: run.notify}\n",
want: "acts as identity",
},
{
name: "profile",
doc: "repository: none\nprofile: build\nsteps:\n - {type: run.notify}\n",
want: "runs under profile",
},
{
name: "required input",
doc: "inputs: {event: {type: object, required: true}}\nrepository: none\nsteps:\n - {type: run.notify}\n",
want: `input "event" is required`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := engine.StartDry(context.Background(), DryRequest{
OrgID: "org", Workflow: "draft", Document: tt.doc,
})
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("StartDry error = %v, want %q", err, tt.want)
}
})
}
}
// TestDryRunNeverExecutesAChangingAction guards the dry run's promise: an
// action that changes something is judged and never executed, however freely
// judgement would let it run, while one that only reads runs for real.
+16 -10
View File
@@ -146,7 +146,7 @@ type progress struct {
// dry run (rc.Dry) judges every action and executes only those that read.
func (d Deps) Compile(doc Document) Workflow {
aliases, withMemory := agentUse(doc)
settleAtEnd := heldAfterLastApproval(doc)
settleAtEnd := settlesAtEnd(doc)
return func(rc *Context, input jsontext.Value) (jsontext.Value, error) {
p := &progress{
input: input, inputs: decodeInputs(input), instructions: doc.Instructions,
@@ -165,8 +165,8 @@ func (d Deps) Compile(doc Document) Workflow {
return nil, err
}
}
// An action held after the last approval step is still put to people
// before the run ends, so nothing judgement held runs unapproved or is
// An action still held when the document ends is put to people before
// the run finishes, so nothing judgement held runs unapproved or is
// silently dropped.
if settleAtEnd && !p.stopped {
if _, err := d.settle(rc, p); err != nil {
@@ -241,19 +241,25 @@ func modelAlias(alias string) string {
return alias
}
// heldAfterLastApproval reports whether a step that can propose an action comes
// after the document's last approval step, or there is none.
func heldAfterLastApproval(doc Document) bool {
held := false
// settlesAtEnd reports whether the run must put held actions to people after
// its last step. It does when a step that can propose an action comes after the
// last approval step, or when that approval step is conditional: a skipped
// approval leaves everything held before it unsettled.
func settlesAtEnd(doc Document) bool {
settled := true
for _, s := range doc.Steps {
switch {
case s.Type == typeApproval && s.When == "" && s.OnFailure != onFailureContinue:
settled = true
case s.Type == typeApproval:
held = false
// A conditional approval may not run, and one that continues past
// its own failure may not settle what it asked, so anything held
// before it stays unsettled unless a later one settles it.
case holds(s.Type):
held = true
settled = false
}
}
return held
return !settled
}
// decodeInputs reads the run's input as the object references resolve inputs
+221
View File
@@ -0,0 +1,221 @@
package runs
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"log/slog"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/memory"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
// TestContinuedFailureDoesNotSwallowSuspension guards the run's stop signals:
// on_failure continue covers a step's own failure, never a suspension, a lost
// lease or a cancellation. A continued suspension would run the rest of the
// workflow while the step is still waiting.
func TestContinuedFailureDoesNotSwallowSuspension(t *testing.T) {
doc, err := ParseDocument(`steps:
- {type: workflow.sleep, on_failure: continue, settings: {for: 1h}}
- {type: run.notify, settings: {title: too soon}}
`)
if err != nil {
t.Fatalf("ParseDocument: %v", err)
}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Judge: &actionJudge{}, Notifier: told}.Compile(doc)
rc := newContext(t.Context(), &fakeWork{}, store.Run{ID: uuid.New()}, uuid.New(), nil)
if _, err := wf(rc, jsontext.Value(`null`)); !errors.Is(err, errSuspended) {
t.Fatalf("run error = %v, want a suspension", err)
}
if len(told.sent) != 0 {
t.Fatalf("a suspended run sent %d notifications", len(told.sent))
}
}
// investigateModel calls the action tool once, reports findings once the tool
// has answered, and answers any later turn (such as recording the cause)
// without a tool call.
type investigateModel struct {
tool string
calls int
}
func (m *investigateModel) Name() string { return "investigate-fake" }
func (m *investigateModel) Chat(_ context.Context, req chat.Request) (chat.Response, error) {
m.calls++
for _, tool := range req.Tools {
if tool.Name != m.tool {
continue
}
last := req.Messages[len(req.Messages)-1]
if last.Role == chat.RoleTool {
return chat.Response{
Role: chat.RoleAssistant, FinishReason: "stop",
Content: "web-1 is out of disk; restart it.",
}, nil
}
return chat.Response{
Role: chat.RoleAssistant, FinishReason: "tool_calls",
ToolCalls: []chat.ToolCall{{ID: "call-1", Name: m.tool, Arguments: `{"target":"web-1"}`}},
}, nil
}
return chat.Response{Role: chat.RoleAssistant, FinishReason: "stop", Content: "no incident recorded"}, nil
}
func (m *investigateModel) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
return nil, errors.New("not streamed")
}
// investigateSettings resolves the default agent model and offers the fake
// model to every agent step.
type investigateSettings struct {
onePersonPath
model chat.Provider
}
func (s investigateSettings) AgentModelForAlias(context.Context, string) (governance.Model, error) {
return governance.Model{Model: "test"}, nil
}
func (s investigateSettings) AgentProvider(context.Context, governance.Model) (chat.Provider, chat.ModelInfo, error) {
return s.model, chat.ModelInfo{}, nil
}
func (investigateSettings) AttemptsPerRun(context.Context) (int, error) { return 8, nil }
// EscalationPath waits long enough that the first attempt suspends, as a
// person who has not answered yet does.
func (investigateSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: "p", DeadlineMS: time.Hour.Milliseconds()}}, nil
}
// investigateBox declares one action so the model has something to call. The
// judge decides it, so the connector itself is never invoked.
type investigateBox struct{}
func (investigateBox) Declarations() []connectors.Declaration {
return []connectors.Declaration{{
Connector: "host", Kind: connectors.KindAction, Name: "restart",
Description: "Restart a host",
Schema: jsontext.Value(`{"type":"object","properties":{"target":{"type":"string"}},"required":["target"]}`),
}}
}
func (investigateBox) Tool(string) (connectors.Tool, bool) { return nil, true }
// heldJudge is actionJudge with the real Execute contract for held actions: a
// held action is not run by the agent's tool call, only by an approval.
type heldJudge struct{ actionJudge }
func (j *heldJudge) Execute(ctx context.Context, d judgement.Decision, input jsontext.Value, key string) (string, error) {
if !d.Level.Runs() {
return "", judgement.ErrHeld
}
return j.actionJudge.Execute(ctx, d, input, key)
}
// investigateMemory records nothing; the workflow only needs the finish step.
type investigateMemory struct{}
func (investigateMemory) Past(context.Context, uuid.UUID, memory.Target) ([]memory.Incident, error) {
return nil, nil
}
func (investigateMemory) Record(context.Context, uuid.UUID, jsontext.Value, memory.Target, string, string) error {
return nil
}
func (investigateMemory) Finish(context.Context, uuid.UUID, []memory.Action, int) (memory.Review, error) {
return memory.Review{}, nil
}
// TestInvestigateDocumentSuspendsAndResumes guards the document the binary
// ships: it investigates the event, holds its action for approval, suspends
// while the person decides, and on resume replays the agent and the request
// instead of asking or running anything twice.
func TestInvestigateDocumentSuspendsAndResumes(t *testing.T) {
doc, err := ParseDocument(InvestigateDocument())
if err != nil {
t.Fatalf("the shipped investigate document doesn't parse: %v", err)
}
model := &investigateModel{tool: toolName("host", "restart")}
judge := &heldJudge{actionJudge{levels: map[string]judgement.Level{"restart": judgement.ProposeAndWait}}}
asked := &askCounter{}
told := &notices{}
deps := Deps{
Settings: investigateSettings{model: model}, Toolbox: investigateBox{},
Judge: judge, Approver: asked, Notifier: told, Memory: investigateMemory{},
}
run := store.Run{
ID: uuid.New(), OrgID: "org", Workflow: "investigate", Version: 1,
Input: jsontext.Value(`{"event":{"fields":{"host":"web-1"}}}`),
}
// The first attempt proposes the action, asks once and suspends on the
// person's answer.
first := &fakeWork{}
engine := newEngine(nil, first, slog.New(slog.DiscardHandler), Options{})
useWorkflow(engine, deps.Compile(doc))
engine.execute(t.Context(), run, uuid.New())
if len(first.finished) != 0 {
t.Fatalf("the first attempt finished %v instead of suspending", first.finished)
}
if asked.asked != 1 || judge.executed != 0 {
t.Fatalf("first attempt asked %d and executed %d, want 1 and 0", asked.asked, judge.executed)
}
// The resumed attempt replays the agent and the request, takes the answer,
// runs the action once and reports one notification.
calls := model.calls
resumed := &signalAfter{steps: first.steps, arrives: 1}
engine = newEngine(nil, resumed, slog.New(slog.DiscardHandler), Options{})
useWorkflow(engine, deps.Compile(doc))
engine.execute(t.Context(), run, uuid.New())
if len(resumed.finished) != 1 || resumed.finished[0] != store.StateSucceeded {
t.Fatalf("resumed run finished %v, want one success", resumed.finished)
}
if asked.asked != 1 {
t.Fatalf("the resumed run asked %d times in total, want the first attempt's one", asked.asked)
}
if judge.executed != 1 {
t.Fatalf("the resumed run executed the action %d times, want once", judge.executed)
}
if model.calls != calls {
t.Fatalf("the resumed run called the model %d times, want none", model.calls-calls)
}
if len(told.sent) != 1 {
t.Fatalf("the run sent %d notifications, want one", len(told.sent))
}
if body := told.sent[0].Body; !strings.Contains(body, "restart") || !strings.Contains(body, approvals.StateApproved) {
t.Fatalf("the notification doesn't report the approved action: %q", body)
}
asks := 0
for _, s := range resumed.steps {
if strings.HasPrefix(s.Name, "ask:") {
asks++
}
}
if asks != 1 {
t.Fatalf("the run recorded %d approval requests, want one", asks)
}
var result AgentResult
if err := json.Unmarshal(resumed.results[0], &result); err != nil {
t.Fatalf("run result: %v", err)
}
if result.Answer == "" {
t.Fatalf("run result has no findings: %s", resumed.results[0])
}
}
+8 -6
View File
@@ -7,8 +7,6 @@ import (
"strings"
"gopkg.in/yaml.v3"
"github.com/samcharles93/archie-core/internal/platform/stableid"
)
// StepResult is the output later steps can reference as
@@ -52,14 +50,18 @@ func (s *refScope) has(id string) bool {
return ok
}
// checkStepID requires a declared step id to be a stable identifier and
// unique within the workflow.
// stepIDPattern is narrower than a stable identifier: a step id is read inside
// a dotted reference, so a dot in the id would make the reference ambiguous.
var stepIDPattern = regexp.MustCompile(`^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$`)
// checkStepID requires a declared step id to be unique within the workflow and
// readable inside a dotted reference.
func checkStepID(id string, seen *refScope) error {
if id == "" {
return nil
}
if !stableid.Valid(id) {
return fmt.Errorf("step id %q is not a stable identifier: lowercase letters and digits, joined by dots or hyphens", id)
if !stepIDPattern.MatchString(id) {
return fmt.Errorf("step id %q must be lowercase letters and digits, joined by hyphens", id)
}
if seen.has(id) {
return fmt.Errorf("step id %q is declared twice", id)
+11 -8
View File
@@ -108,7 +108,7 @@ func (e *Engine) steps() (Deps, error) {
// what the latest version already is, and returns the version in force. A body
// that doesn't validate is stored as nothing.
func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int, error) {
if err := e.validateDocument(ctx, body); err != nil {
if _, err := e.validateDocument(ctx, body); err != nil {
return 0, err
}
latest, err := e.db.Workflow(ctx, orgID, name, 0)
@@ -125,10 +125,10 @@ func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int
return w.Version, nil
}
func (e *Engine) validateDocument(ctx context.Context, body string) error {
func (e *Engine) validateDocument(ctx context.Context, body string) (Document, error) {
doc, err := ParseDocument(body)
if err != nil {
return err
return Document{}, err
}
var problems []Problem
for i, step := range doc.Steps {
@@ -137,26 +137,26 @@ func (e *Engine) validateDocument(ctx context.Context, body string) error {
}
var s agentSettings
if err := decodeSettings(step.Settings, &s); err != nil {
return err
return Document{}, err
}
if modelAlias(s.Model) == "" {
continue
}
deps, err := e.steps()
if err != nil {
return err
return Document{}, err
}
if _, err := deps.Settings.AgentModelForAlias(ctx, s.Model); err != nil {
if !errors.Is(err, governance.ErrNoAgentModel) {
return err
return Document{}, err
}
problems = append(problems, Problem{Path: fmt.Sprintf("steps[%d].settings.model", i), Message: fmt.Sprintf("unknown agent model alias %q", s.Model)})
}
}
if len(problems) > 0 {
return &InvalidError{Problems: problems}
return Document{}, &InvalidError{Problems: problems}
}
return nil
return doc, nil
}
// Has reports whether a workflow is stored under name.
@@ -215,6 +215,9 @@ func (e *Engine) CheckTrigger(ctx context.Context, orgID, workflow string) error
if err != nil {
return refuse("workflow %q is not valid: %v", workflow, err)
}
if err := doc.unsupported(); err != nil {
return refuse("workflow %q can't start: %v", workflow, err)
}
if missing := doc.requires("event"); len(missing) > 0 {
return refuse("workflow %q requires %s, and a trigger supplies only event", workflow, strings.Join(missing, ", "))
}
+32 -3
View File
@@ -53,11 +53,19 @@ func (j *actionJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision,
return j.decisions, nil
}
// askCounter approves every request at once and counts them.
type askCounter struct{ asked int }
// askCounter approves every request at once and counts them. failFirst makes
// that many requests fail first, as an approvals store that is briefly down.
type askCounter struct {
asked int
failFirst int
}
func (a *askCounter) Ask(_ context.Context, _ uuid.UUID, _ string, d judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
a.asked++
if a.failFirst > 0 {
a.failFirst--
return approvals.Request{}, errors.New("approvals unavailable")
}
return approvals.Request{ID: uuid.New(), State: approvals.StatePending, Decision: d}, nil
}
@@ -106,6 +114,7 @@ func TestDocumentStepsRun(t *testing.T) {
doc string
levels map[string]judgement.Level
fail error
failFirst int
wantErr bool
wantInput string
wantAssessed int
@@ -153,6 +162,26 @@ steps:
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "a held action is settled when the last approval step is skipped",
doc: `inputs: {skip: {type: bool}}
steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, when: "inputs.skip"}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
wantAssessed: 2, wantAsked: 1, wantExecuted: 1,
},
{
name: "an approval that fails and continues is settled before the run ends",
doc: `steps:
- {type: connector.action, settings: {connector: host, name: clean}}
- {type: approval.wait, on_failure: continue}
`,
levels: map[string]judgement.Level{"clean": judgement.ProposeAndWait},
failFirst: 1,
wantAssessed: 2, wantAsked: 2, wantExecuted: 1,
},
{
name: "a later approval step asks only about what was held since the last",
doc: `steps:
@@ -181,7 +210,7 @@ steps:
t.Fatalf("ParseDocument: %v", err)
}
judge := &actionJudge{levels: tt.levels, fail: tt.fail}
ask := &askCounter{}
ask := &askCounter{failFirst: tt.failFirst}
told := &notices{}
wf := Deps{Settings: onePersonPath{}, Toolbox: anyBox{}, Judge: judge, Approver: ask, Notifier: told}.Compile(doc)
work := &signalAfter{arrives: 1}