feat(runs): an action started before a crash is reported unknown on resume and never run again, and the owner is told (archie #2015)

This commit is contained in:
sam committed 2026-11-16 05:18:05 +11:00
1 parent 55077620bb
commit 85e6f8eb40
4 files changed
+214 -18

No files matched your search

+6
View File
@@ -2,6 +2,7 @@ package runs
import (
"context"
"errors"
"github.com/samcharles93/ai-sdk/agentloop"
"github.com/samcharles93/ai-sdk/chat"
@@ -62,6 +63,11 @@ func (rc *Context) Agent(req AgentRequest) (AgentResult, error) {
if out.Completed && len(res.Steps) > 0 {
out.Answer = res.Steps[len(res.Steps)-1].Text
}
// An action that may have taken effect parks the loop. The unknown outcome
// is the result, so the workflow can tell the owner.
if errors.Is(err, core.ErrOutcomeUnknown) && out.StopReason == agentloop.StopOutcomeUnknown {
return out, nil
}
if err != nil {
return out, err
}
+35 -1
View File
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/google/uuid"
@@ -40,14 +41,47 @@ type Context struct {
token uuid.UUID
saved map[int]store.Step
next int
// started holds the keys of effects an earlier attempt began; new markers
// are numbered from nextMarker.
started map[string]bool
nextMarker int
}
// markerBase is where started markers are numbered, far above any step a
// workflow takes, so they never shift the positions a replay depends on.
const markerBase = 1 << 20
const startedPrefix = "started:"
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
saved := make(map[int]store.Step, len(steps))
started := map[string]bool{}
nextMarker := markerBase
for _, s := range steps {
saved[s.Seq] = s
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
started[key] = true
nextMarker = max(nextMarker, s.Seq+1)
}
}
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved}
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
}
// Begin durably records that the effect behind key is about to happen, before
// it does. It reports true when an earlier attempt already began it: that
// attempt may have died after the effect and before its result was recorded,
// so the caller must not run it again and treats its outcome as unknown (R15).
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
if rc.started[key] {
return true, nil
}
seq := rc.nextMarker
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
return false, err
}
rc.started[key] = true
rc.nextMarker = seq + 1
return false, nil
}
// RunID is the run's id.
+77 -17
View File
@@ -78,20 +78,10 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
if err != nil {
return nil, err
}
// Pinned here, like the model: a run that waits keeps the deadline it
// started with, whatever the setting becomes meanwhile. A step's value
// is encoded, so it travels as the duration's own text.
deadlineText, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
d, err := settings.ApprovalDeadline(ctx)
return d.String(), err
})
deadline, err := pinnedDeadline(rc, settings)
if err != nil {
return nil, err
}
deadline, err := time.ParseDuration(deadlineText)
if err != nil {
return nil, fmt.Errorf("approval deadline %q: %w", deadlineText, err)
}
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
return settings.RecurrenceThreshold(ctx)
})
@@ -115,7 +105,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
ModelInfo: info,
System: prompts.Investigate(),
Prompt: string(input),
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
Budget: investigateBudget,
})
if err != nil {
@@ -123,10 +113,13 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
}
if res, err = followThrough(rc, judge, decls, input, res, AgentRequest{
Provider: provider, Model: model.Model, ModelInfo: info,
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
}); err != nil {
return nil, err
}
if res.StopReason == agentloop.StopOutcomeUnknown {
return stopUnknown(rc, notify, res)
}
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
return nil, err
}
@@ -154,6 +147,54 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
}
}
// pinnedDeadline is how long an approval request waits. It is pinned in a step,
// like the model: a run that waits keeps the deadline it started with, whatever
// the setting becomes meanwhile. A step's value is encoded, so it travels as
// the duration's own text.
func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) {
text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
d, err := settings.ApprovalDeadline(ctx)
return d.String(), err
})
if err != nil {
return 0, err
}
d, err := time.ParseDuration(text)
if err != nil {
return 0, fmt.Errorf("approval deadline %q: %w", text, err)
}
return d, nil
}
// stopUnknown ends an investigation that stopped on an action whose outcome
// can't be confirmed. Nothing further is proposed or run; the owner is told.
func stopUnknown(rc *Context, notify Notifier, res AgentResult) (jsontext.Value, error) {
if _, err := rc.Step("notify", func(ctx context.Context, key string) (bool, error) {
return true, notify.Notify(ctx, findings(rc.RunID(), key, res, nil, memory.Review{}))
}); err != nil {
return nil, err
}
out, err := json.Marshal(res)
if err != nil {
return nil, fmt.Errorf("encode findings: %w", err)
}
return out, fmt.Errorf("investigation stopped: %s", res.StopReason)
}
// begun records that an action is about to run. It fails with an unknown
// outcome when an earlier attempt already started it, so the action is not run
// again.
func begun(ctx context.Context, rc *Context, key string) error {
started, err := rc.Begin(ctx, key)
switch {
case err != nil:
return err
case started:
return fmt.Errorf("%w: %w: an earlier attempt started it and did not record how it ended", ErrOutcomeUnknown, core.ErrOutcomeUnknown)
}
return nil
}
// followThroughBudget bounds the turn that has the agent call what it only
// described.
var followThroughBudget = agentloop.Budget{MaxSteps: 8, MaxTokens: 60_000, WallClock: 5 * time.Minute}
@@ -186,6 +227,9 @@ func followThrough(rc *Context, judge Judge, decls []connectors.Declaration, inp
}
req.Name, req.System, req.Prompt, req.Budget = "follow-through", prompts.Investigate()+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
again, err := rc.Agent(req)
if err == nil && again.StopReason == agentloop.StopOutcomeUnknown {
return again, nil
}
if err != nil || !again.Completed {
// The first findings stand when the extra turn doesn't finish.
return res, err
@@ -340,11 +384,17 @@ func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approval
case fresh.Level == judgement.Never:
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
case fresh.Level.Runs():
if err := begun(ctx, rc, key); err != nil {
return "", err
}
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
return out, unknownOutcome(err)
case fresh.Level > d.Level:
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
}
if err := begun(ctx, rc, key); err != nil {
return "", err
}
out, err := judge.ExecuteApproved(ctx, d, answer.Approval(d), d.Action.Input, key)
return out, unknownOutcome(err)
})
@@ -372,6 +422,11 @@ func waitKey(key string) string { return "approval:" + key }
// findings is the notification an investigation ends with, finished or not.
func findings(runID uuid.UUID, key string, res AgentResult, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
if res.StopReason == agentloop.StopOutcomeUnknown {
n.Title = "An action's outcome is unknown"
n.Body = "An action was started and its outcome can't be confirmed. It may have taken effect and is not run again; check its target. " + res.Detail
return n
}
if !res.Completed {
n.Title = "Investigation stopped"
n.Body = fmt.Sprintf("The investigation stopped before it finished (%s). %s", res.StopReason, res.Detail)
@@ -426,7 +481,7 @@ var unsafeToolChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
// toolSet turns pinned declarations into the agent's tools. Each tool calls
// its connector or MCP server by name when the model asks, so one that has
// since stopped fails that call rather than the run.
func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Declaration) core.ToolSet {
func toolSet(box Toolbox, judge Judge, rc *Context, decls []connectors.Declaration) core.ToolSet {
set := core.ToolSet{}
for _, d := range decls {
name := toolName(d.Connector, d.Name)
@@ -435,14 +490,14 @@ func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Decla
Description: d.Description,
Parameters: []byte(d.Schema),
Execute: func(ctx context.Context, input string) (string, error) {
return call(ctx, box, judge, runID, d, jsontext.Value(input))
return call(ctx, box, judge, rc, d, jsontext.Value(input))
},
}
}
return set
}
func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d connectors.Declaration, input jsontext.Value) (string, error) {
func call(ctx context.Context, box Toolbox, judge Judge, rc *Context, d connectors.Declaration, input jsontext.Value) (string, error) {
tool, ok := box.Tool(d.Connector)
if !ok {
return "", fmt.Errorf("%q isn't running", d.Connector)
@@ -461,10 +516,15 @@ func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d conn
// is proposed to a person when the investigation ends, so it comes back to
// the agent as something waiting rather than a failure.
key := core.IdempotencyKey(ctx)
decision, err := judge.Assess(ctx, judgement.Request{RunID: runID, Key: key, Connector: d.Connector, Action: d.Name, Input: input})
decision, err := judge.Assess(ctx, judgement.Request{RunID: rc.RunID(), Key: key, Connector: d.Connector, Action: d.Name, Input: input})
if err != nil {
return "", err
}
if decision.Level.Runs() {
if err := begun(ctx, rc, key); err != nil {
return "", err
}
}
out, err := judge.Execute(ctx, decision, input, key)
if errors.Is(err, judgement.ErrHeld) {
return fmt.Sprintf("held at %s for a person's approval: %s. It is asked when your investigation ends, so don't propose it again.", decision.Level, findingsOf(decision)), nil
+96
View File
@@ -0,0 +1,96 @@
package runs
import (
"context"
"encoding/json/jsontext"
"errors"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/runs/internal/store"
)
type countingJudge struct {
level judgement.Level
executed int
}
func (j *countingJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
return judgement.Decision{Key: req.Key, Level: j.level, Action: judgement.Action{Connector: req.Connector, Name: req.Action, Input: req.Input}}, nil
}
func (j *countingJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *countingJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
j.executed++
return "done", nil
}
func (j *countingJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
return nil, nil
}
type stubBox struct{ tool connectors.Tool }
func (stubBox) Declarations() []connectors.Declaration { return nil }
func (b stubBox) Tool(string) (connectors.Tool, bool) { return b.tool, true }
// TestActionStartedBeforeItsResultIsNeverRunAgain guards R15: a crash between
// a connector's effect and its recorded result leaves a started marker and no
// result, and the resumed run must report the outcome as unknown instead of
// running the effect a second time.
func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) {
runID := uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")
decl := connectors.Declaration{Connector: "ops", Kind: connectors.KindAction, Name: "restart"}
marker := func(key string) []store.Step {
return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}}
}
agentKey := runID.String() + ":investigate:3"
approvedKey := runID.String() + ":1"
tests := []struct {
name string
steps []store.Step
approved bool
wantExecuted int
wantUnknown bool
}{
{name: "agent action with no marker runs", wantExecuted: 1},
{name: "agent action with a marker is unknown", steps: marker(agentKey), wantUnknown: true},
{name: "approved action with no marker runs", approved: true, wantExecuted: 1},
{name: "approved action with a marker is unknown", steps: marker(approvedKey), approved: true, wantUnknown: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
work := &fakeWork{steps: tt.steps}
run := store.Run{ID: runID, OrgID: "org"}
rc := newContext(t.Context(), work, run, uuid.New(), tt.steps)
judge := &countingJudge{level: judgement.ActAndTell}
var err error
if tt.approved {
d := judgement.Decision{Key: "k", Level: judgement.ProposeAndWait, Action: judgement.Action{Connector: "ops", Name: "restart"}}
_, err = runApproved(rc, judge, d, approvals.Answer{Approved: true})
} else {
ctx := core.WithIdempotencyKey(t.Context(), agentKey)
_, err = call(ctx, stubBox{}, judge, rc, decl, jsontext.Value(`{}`))
}
if got := errors.Is(err, ErrOutcomeUnknown) && errors.Is(err, core.ErrOutcomeUnknown); got != tt.wantUnknown {
t.Fatalf("unknown outcome = %v (err %v), want %v", got, err, tt.wantUnknown)
}
if judge.executed != tt.wantExecuted {
t.Fatalf("action executed %d times, want %d", judge.executed, tt.wantExecuted)
}
})
}
}