feat(runs): an action started before a crash is reported unknown on resume and never run again, and the owner is told (archie #2015)
This commit is contained in:
4 files changed
+214
-18
No files matched your search
@@ -2,6 +2,7 @@ package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/agentloop"
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
@@ -62,6 +63,11 @@ func (rc *Context) Agent(req AgentRequest) (AgentResult, error) {
|
||||
if out.Completed && len(res.Steps) > 0 {
|
||||
out.Answer = res.Steps[len(res.Steps)-1].Text
|
||||
}
|
||||
// An action that may have taken effect parks the loop. The unknown outcome
|
||||
// is the result, so the workflow can tell the owner.
|
||||
if errors.Is(err, core.ErrOutcomeUnknown) && out.StopReason == agentloop.StopOutcomeUnknown {
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
|
||||
+35
-1
@@ -8,6 +8,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -40,14 +41,47 @@ type Context struct {
|
||||
token uuid.UUID
|
||||
saved map[int]store.Step
|
||||
next int
|
||||
// started holds the keys of effects an earlier attempt began; new markers
|
||||
// are numbered from nextMarker.
|
||||
started map[string]bool
|
||||
nextMarker int
|
||||
}
|
||||
|
||||
// markerBase is where started markers are numbered, far above any step a
|
||||
// workflow takes, so they never shift the positions a replay depends on.
|
||||
const markerBase = 1 << 20
|
||||
|
||||
const startedPrefix = "started:"
|
||||
|
||||
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
||||
saved := make(map[int]store.Step, len(steps))
|
||||
started := map[string]bool{}
|
||||
nextMarker := markerBase
|
||||
for _, s := range steps {
|
||||
saved[s.Seq] = s
|
||||
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
|
||||
started[key] = true
|
||||
nextMarker = max(nextMarker, s.Seq+1)
|
||||
}
|
||||
}
|
||||
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved}
|
||||
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
|
||||
}
|
||||
|
||||
// Begin durably records that the effect behind key is about to happen, before
|
||||
// it does. It reports true when an earlier attempt already began it: that
|
||||
// attempt may have died after the effect and before its result was recorded,
|
||||
// so the caller must not run it again and treats its outcome as unknown (R15).
|
||||
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
|
||||
if rc.started[key] {
|
||||
return true, nil
|
||||
}
|
||||
seq := rc.nextMarker
|
||||
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
|
||||
return false, err
|
||||
}
|
||||
rc.started[key] = true
|
||||
rc.nextMarker = seq + 1
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// RunID is the run's id.
|
||||
|
||||
+77
-17
@@ -78,20 +78,10 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Pinned here, like the model: a run that waits keeps the deadline it
|
||||
// started with, whatever the setting becomes meanwhile. A step's value
|
||||
// is encoded, so it travels as the duration's own text.
|
||||
deadlineText, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
|
||||
d, err := settings.ApprovalDeadline(ctx)
|
||||
return d.String(), err
|
||||
})
|
||||
deadline, err := pinnedDeadline(rc, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline, err := time.ParseDuration(deadlineText)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("approval deadline %q: %w", deadlineText, err)
|
||||
}
|
||||
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
||||
return settings.RecurrenceThreshold(ctx)
|
||||
})
|
||||
@@ -115,7 +105,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
ModelInfo: info,
|
||||
System: prompts.Investigate(),
|
||||
Prompt: string(input),
|
||||
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
|
||||
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||
Budget: investigateBudget,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -123,10 +113,13 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
}
|
||||
if res, err = followThrough(rc, judge, decls, input, res, AgentRequest{
|
||||
Provider: provider, Model: model.Model, ModelInfo: info,
|
||||
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
|
||||
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||
return stopUnknown(rc, notify, res)
|
||||
}
|
||||
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -154,6 +147,54 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
||||
}
|
||||
}
|
||||
|
||||
// pinnedDeadline is how long an approval request waits. It is pinned in a step,
|
||||
// like the model: a run that waits keeps the deadline it started with, whatever
|
||||
// the setting becomes meanwhile. A step's value is encoded, so it travels as
|
||||
// the duration's own text.
|
||||
func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) {
|
||||
text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
|
||||
d, err := settings.ApprovalDeadline(ctx)
|
||||
return d.String(), err
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
d, err := time.ParseDuration(text)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("approval deadline %q: %w", text, err)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// stopUnknown ends an investigation that stopped on an action whose outcome
|
||||
// can't be confirmed. Nothing further is proposed or run; the owner is told.
|
||||
func stopUnknown(rc *Context, notify Notifier, res AgentResult) (jsontext.Value, error) {
|
||||
if _, err := rc.Step("notify", func(ctx context.Context, key string) (bool, error) {
|
||||
return true, notify.Notify(ctx, findings(rc.RunID(), key, res, nil, memory.Review{}))
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode findings: %w", err)
|
||||
}
|
||||
return out, fmt.Errorf("investigation stopped: %s", res.StopReason)
|
||||
}
|
||||
|
||||
// begun records that an action is about to run. It fails with an unknown
|
||||
// outcome when an earlier attempt already started it, so the action is not run
|
||||
// again.
|
||||
func begun(ctx context.Context, rc *Context, key string) error {
|
||||
started, err := rc.Begin(ctx, key)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case started:
|
||||
return fmt.Errorf("%w: %w: an earlier attempt started it and did not record how it ended", ErrOutcomeUnknown, core.ErrOutcomeUnknown)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// followThroughBudget bounds the turn that has the agent call what it only
|
||||
// described.
|
||||
var followThroughBudget = agentloop.Budget{MaxSteps: 8, MaxTokens: 60_000, WallClock: 5 * time.Minute}
|
||||
@@ -186,6 +227,9 @@ func followThrough(rc *Context, judge Judge, decls []connectors.Declaration, inp
|
||||
}
|
||||
req.Name, req.System, req.Prompt, req.Budget = "follow-through", prompts.Investigate()+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
||||
again, err := rc.Agent(req)
|
||||
if err == nil && again.StopReason == agentloop.StopOutcomeUnknown {
|
||||
return again, nil
|
||||
}
|
||||
if err != nil || !again.Completed {
|
||||
// The first findings stand when the extra turn doesn't finish.
|
||||
return res, err
|
||||
@@ -340,11 +384,17 @@ func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approval
|
||||
case fresh.Level == judgement.Never:
|
||||
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
|
||||
case fresh.Level.Runs():
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
|
||||
return out, unknownOutcome(err)
|
||||
case fresh.Level > d.Level:
|
||||
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
|
||||
}
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := judge.ExecuteApproved(ctx, d, answer.Approval(d), d.Action.Input, key)
|
||||
return out, unknownOutcome(err)
|
||||
})
|
||||
@@ -372,6 +422,11 @@ func waitKey(key string) string { return "approval:" + key }
|
||||
// findings is the notification an investigation ends with, finished or not.
|
||||
func findings(runID uuid.UUID, key string, res AgentResult, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
||||
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
|
||||
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||
n.Title = "An action's outcome is unknown"
|
||||
n.Body = "An action was started and its outcome can't be confirmed. It may have taken effect and is not run again; check its target. " + res.Detail
|
||||
return n
|
||||
}
|
||||
if !res.Completed {
|
||||
n.Title = "Investigation stopped"
|
||||
n.Body = fmt.Sprintf("The investigation stopped before it finished (%s). %s", res.StopReason, res.Detail)
|
||||
@@ -426,7 +481,7 @@ var unsafeToolChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
|
||||
// toolSet turns pinned declarations into the agent's tools. Each tool calls
|
||||
// its connector or MCP server by name when the model asks, so one that has
|
||||
// since stopped fails that call rather than the run.
|
||||
func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Declaration) core.ToolSet {
|
||||
func toolSet(box Toolbox, judge Judge, rc *Context, decls []connectors.Declaration) core.ToolSet {
|
||||
set := core.ToolSet{}
|
||||
for _, d := range decls {
|
||||
name := toolName(d.Connector, d.Name)
|
||||
@@ -435,14 +490,14 @@ func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Decla
|
||||
Description: d.Description,
|
||||
Parameters: []byte(d.Schema),
|
||||
Execute: func(ctx context.Context, input string) (string, error) {
|
||||
return call(ctx, box, judge, runID, d, jsontext.Value(input))
|
||||
return call(ctx, box, judge, rc, d, jsontext.Value(input))
|
||||
},
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d connectors.Declaration, input jsontext.Value) (string, error) {
|
||||
func call(ctx context.Context, box Toolbox, judge Judge, rc *Context, d connectors.Declaration, input jsontext.Value) (string, error) {
|
||||
tool, ok := box.Tool(d.Connector)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%q isn't running", d.Connector)
|
||||
@@ -461,10 +516,15 @@ func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d conn
|
||||
// is proposed to a person when the investigation ends, so it comes back to
|
||||
// the agent as something waiting rather than a failure.
|
||||
key := core.IdempotencyKey(ctx)
|
||||
decision, err := judge.Assess(ctx, judgement.Request{RunID: runID, Key: key, Connector: d.Connector, Action: d.Name, Input: input})
|
||||
decision, err := judge.Assess(ctx, judgement.Request{RunID: rc.RunID(), Key: key, Connector: d.Connector, Action: d.Name, Input: input})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if decision.Level.Runs() {
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
out, err := judge.Execute(ctx, decision, input, key)
|
||||
if errors.Is(err, judgement.ErrHeld) {
|
||||
return fmt.Sprintf("held at %s for a person's approval: %s. It is asked when your investigation ends, so don't propose it again.", decision.Level, findingsOf(decision)), nil
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/approvals"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
type countingJudge struct {
|
||||
level judgement.Level
|
||||
executed int
|
||||
}
|
||||
|
||||
func (j *countingJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
||||
return judgement.Decision{Key: req.Key, Level: j.level, Action: judgement.Action{Connector: req.Connector, Name: req.Action, Input: req.Input}}, nil
|
||||
}
|
||||
|
||||
func (j *countingJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
|
||||
j.executed++
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func (j *countingJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
|
||||
j.executed++
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func (j *countingJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type stubBox struct{ tool connectors.Tool }
|
||||
|
||||
func (stubBox) Declarations() []connectors.Declaration { return nil }
|
||||
|
||||
func (b stubBox) Tool(string) (connectors.Tool, bool) { return b.tool, true }
|
||||
|
||||
// TestActionStartedBeforeItsResultIsNeverRunAgain guards R15: a crash between
|
||||
// a connector's effect and its recorded result leaves a started marker and no
|
||||
// result, and the resumed run must report the outcome as unknown instead of
|
||||
// running the effect a second time.
|
||||
func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) {
|
||||
runID := uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")
|
||||
decl := connectors.Declaration{Connector: "ops", Kind: connectors.KindAction, Name: "restart"}
|
||||
marker := func(key string) []store.Step {
|
||||
return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}}
|
||||
}
|
||||
agentKey := runID.String() + ":investigate:3"
|
||||
approvedKey := runID.String() + ":1"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
steps []store.Step
|
||||
approved bool
|
||||
wantExecuted int
|
||||
wantUnknown bool
|
||||
}{
|
||||
{name: "agent action with no marker runs", wantExecuted: 1},
|
||||
{name: "agent action with a marker is unknown", steps: marker(agentKey), wantUnknown: true},
|
||||
{name: "approved action with no marker runs", approved: true, wantExecuted: 1},
|
||||
{name: "approved action with a marker is unknown", steps: marker(approvedKey), approved: true, wantUnknown: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
work := &fakeWork{steps: tt.steps}
|
||||
run := store.Run{ID: runID, OrgID: "org"}
|
||||
rc := newContext(t.Context(), work, run, uuid.New(), tt.steps)
|
||||
judge := &countingJudge{level: judgement.ActAndTell}
|
||||
|
||||
var err error
|
||||
if tt.approved {
|
||||
d := judgement.Decision{Key: "k", Level: judgement.ProposeAndWait, Action: judgement.Action{Connector: "ops", Name: "restart"}}
|
||||
_, err = runApproved(rc, judge, d, approvals.Answer{Approved: true})
|
||||
} else {
|
||||
ctx := core.WithIdempotencyKey(t.Context(), agentKey)
|
||||
_, err = call(ctx, stubBox{}, judge, rc, decl, jsontext.Value(`{}`))
|
||||
}
|
||||
|
||||
if got := errors.Is(err, ErrOutcomeUnknown) && errors.Is(err, core.ErrOutcomeUnknown); got != tt.wantUnknown {
|
||||
t.Fatalf("unknown outcome = %v (err %v), want %v", got, err, tt.wantUnknown)
|
||||
}
|
||||
if judge.executed != tt.wantExecuted {
|
||||
t.Fatalf("action executed %d times, want %d", judge.executed, tt.wantExecuted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user