feat(runs): an action started before a crash is reported unknown on resume and never run again, and the owner is told (archie #2015)
This commit is contained in:
4 files changed
+214
-18
No files matched your search
@@ -2,6 +2,7 @@ package runs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
|
||||||
"github.com/samcharles93/ai-sdk/agentloop"
|
"github.com/samcharles93/ai-sdk/agentloop"
|
||||||
"github.com/samcharles93/ai-sdk/chat"
|
"github.com/samcharles93/ai-sdk/chat"
|
||||||
@@ -62,6 +63,11 @@ func (rc *Context) Agent(req AgentRequest) (AgentResult, error) {
|
|||||||
if out.Completed && len(res.Steps) > 0 {
|
if out.Completed && len(res.Steps) > 0 {
|
||||||
out.Answer = res.Steps[len(res.Steps)-1].Text
|
out.Answer = res.Steps[len(res.Steps)-1].Text
|
||||||
}
|
}
|
||||||
|
// An action that may have taken effect parks the loop. The unknown outcome
|
||||||
|
// is the result, so the workflow can tell the owner.
|
||||||
|
if errors.Is(err, core.ErrOutcomeUnknown) && out.StopReason == agentloop.StopOutcomeUnknown {
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
|||||||
+35
-1
@@ -8,6 +8,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -40,14 +41,47 @@ type Context struct {
|
|||||||
token uuid.UUID
|
token uuid.UUID
|
||||||
saved map[int]store.Step
|
saved map[int]store.Step
|
||||||
next int
|
next int
|
||||||
|
// started holds the keys of effects an earlier attempt began; new markers
|
||||||
|
// are numbered from nextMarker.
|
||||||
|
started map[string]bool
|
||||||
|
nextMarker int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// markerBase is where started markers are numbered, far above any step a
|
||||||
|
// workflow takes, so they never shift the positions a replay depends on.
|
||||||
|
const markerBase = 1 << 20
|
||||||
|
|
||||||
|
const startedPrefix = "started:"
|
||||||
|
|
||||||
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
func newContext(ctx context.Context, work workerStore, run store.Run, token uuid.UUID, steps []store.Step) *Context {
|
||||||
saved := make(map[int]store.Step, len(steps))
|
saved := make(map[int]store.Step, len(steps))
|
||||||
|
started := map[string]bool{}
|
||||||
|
nextMarker := markerBase
|
||||||
for _, s := range steps {
|
for _, s := range steps {
|
||||||
saved[s.Seq] = s
|
saved[s.Seq] = s
|
||||||
|
if key, ok := strings.CutPrefix(s.Name, startedPrefix); ok {
|
||||||
|
started[key] = true
|
||||||
|
nextMarker = max(nextMarker, s.Seq+1)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved}
|
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Begin durably records that the effect behind key is about to happen, before
|
||||||
|
// it does. It reports true when an earlier attempt already began it: that
|
||||||
|
// attempt may have died after the effect and before its result was recorded,
|
||||||
|
// so the caller must not run it again and treats its outcome as unknown (R15).
|
||||||
|
func (rc *Context) Begin(ctx context.Context, key string) (bool, error) {
|
||||||
|
if rc.started[key] {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
seq := rc.nextMarker
|
||||||
|
if _, err := rc.record(ctx, seq, startedPrefix+key, true, nil); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
rc.started[key] = true
|
||||||
|
rc.nextMarker = seq + 1
|
||||||
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunID is the run's id.
|
// RunID is the run's id.
|
||||||
|
|||||||
+77
-17
@@ -78,20 +78,10 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Pinned here, like the model: a run that waits keeps the deadline it
|
deadline, err := pinnedDeadline(rc, settings)
|
||||||
// started with, whatever the setting becomes meanwhile. A step's value
|
|
||||||
// is encoded, so it travels as the duration's own text.
|
|
||||||
deadlineText, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
|
|
||||||
d, err := settings.ApprovalDeadline(ctx)
|
|
||||||
return d.String(), err
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
deadline, err := time.ParseDuration(deadlineText)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("approval deadline %q: %w", deadlineText, err)
|
|
||||||
}
|
|
||||||
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
||||||
return settings.RecurrenceThreshold(ctx)
|
return settings.RecurrenceThreshold(ctx)
|
||||||
})
|
})
|
||||||
@@ -115,7 +105,7 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
|||||||
ModelInfo: info,
|
ModelInfo: info,
|
||||||
System: prompts.Investigate(),
|
System: prompts.Investigate(),
|
||||||
Prompt: string(input),
|
Prompt: string(input),
|
||||||
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
|
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||||
Budget: investigateBudget,
|
Budget: investigateBudget,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -123,10 +113,13 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
|||||||
}
|
}
|
||||||
if res, err = followThrough(rc, judge, decls, input, res, AgentRequest{
|
if res, err = followThrough(rc, judge, decls, input, res, AgentRequest{
|
||||||
Provider: provider, Model: model.Model, ModelInfo: info,
|
Provider: provider, Model: model.Model, ModelInfo: info,
|
||||||
Tools: withMemory(toolSet(box, judge, rc.RunID(), decls), mem, rc.RunID()),
|
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||||
|
return stopUnknown(rc, notify, res)
|
||||||
|
}
|
||||||
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
|
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -154,6 +147,54 @@ func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, noti
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pinnedDeadline is how long an approval request waits. It is pinned in a step,
|
||||||
|
// like the model: a run that waits keeps the deadline it started with, whatever
|
||||||
|
// the setting becomes meanwhile. A step's value is encoded, so it travels as
|
||||||
|
// the duration's own text.
|
||||||
|
func pinnedDeadline(rc *Context, settings Settings) (time.Duration, error) {
|
||||||
|
text, err := rc.Step("approval-deadline", func(ctx context.Context, _ string) (string, error) {
|
||||||
|
d, err := settings.ApprovalDeadline(ctx)
|
||||||
|
return d.String(), err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(text)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("approval deadline %q: %w", text, err)
|
||||||
|
}
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopUnknown ends an investigation that stopped on an action whose outcome
|
||||||
|
// can't be confirmed. Nothing further is proposed or run; the owner is told.
|
||||||
|
func stopUnknown(rc *Context, notify Notifier, res AgentResult) (jsontext.Value, error) {
|
||||||
|
if _, err := rc.Step("notify", func(ctx context.Context, key string) (bool, error) {
|
||||||
|
return true, notify.Notify(ctx, findings(rc.RunID(), key, res, nil, memory.Review{}))
|
||||||
|
}); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out, err := json.Marshal(res)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("encode findings: %w", err)
|
||||||
|
}
|
||||||
|
return out, fmt.Errorf("investigation stopped: %s", res.StopReason)
|
||||||
|
}
|
||||||
|
|
||||||
|
// begun records that an action is about to run. It fails with an unknown
|
||||||
|
// outcome when an earlier attempt already started it, so the action is not run
|
||||||
|
// again.
|
||||||
|
func begun(ctx context.Context, rc *Context, key string) error {
|
||||||
|
started, err := rc.Begin(ctx, key)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return err
|
||||||
|
case started:
|
||||||
|
return fmt.Errorf("%w: %w: an earlier attempt started it and did not record how it ended", ErrOutcomeUnknown, core.ErrOutcomeUnknown)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// followThroughBudget bounds the turn that has the agent call what it only
|
// followThroughBudget bounds the turn that has the agent call what it only
|
||||||
// described.
|
// described.
|
||||||
var followThroughBudget = agentloop.Budget{MaxSteps: 8, MaxTokens: 60_000, WallClock: 5 * time.Minute}
|
var followThroughBudget = agentloop.Budget{MaxSteps: 8, MaxTokens: 60_000, WallClock: 5 * time.Minute}
|
||||||
@@ -186,6 +227,9 @@ func followThrough(rc *Context, judge Judge, decls []connectors.Declaration, inp
|
|||||||
}
|
}
|
||||||
req.Name, req.System, req.Prompt, req.Budget = "follow-through", prompts.Investigate()+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
req.Name, req.System, req.Prompt, req.Budget = "follow-through", prompts.Investigate()+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
||||||
again, err := rc.Agent(req)
|
again, err := rc.Agent(req)
|
||||||
|
if err == nil && again.StopReason == agentloop.StopOutcomeUnknown {
|
||||||
|
return again, nil
|
||||||
|
}
|
||||||
if err != nil || !again.Completed {
|
if err != nil || !again.Completed {
|
||||||
// The first findings stand when the extra turn doesn't finish.
|
// The first findings stand when the extra turn doesn't finish.
|
||||||
return res, err
|
return res, err
|
||||||
@@ -340,11 +384,17 @@ func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approval
|
|||||||
case fresh.Level == judgement.Never:
|
case fresh.Level == judgement.Never:
|
||||||
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
|
return "", fmt.Errorf("refused after approval: %s", fresh.Reason)
|
||||||
case fresh.Level.Runs():
|
case fresh.Level.Runs():
|
||||||
|
if err := begun(ctx, rc, key); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
|
out, err := judge.Execute(ctx, fresh, d.Action.Input, key)
|
||||||
return out, unknownOutcome(err)
|
return out, unknownOutcome(err)
|
||||||
case fresh.Level > d.Level:
|
case fresh.Level > d.Level:
|
||||||
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
|
return "", fmt.Errorf("judgement got worse after approval: now %s", fresh.Level)
|
||||||
}
|
}
|
||||||
|
if err := begun(ctx, rc, key); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
out, err := judge.ExecuteApproved(ctx, d, answer.Approval(d), d.Action.Input, key)
|
out, err := judge.ExecuteApproved(ctx, d, answer.Approval(d), d.Action.Input, key)
|
||||||
return out, unknownOutcome(err)
|
return out, unknownOutcome(err)
|
||||||
})
|
})
|
||||||
@@ -372,6 +422,11 @@ func waitKey(key string) string { return "approval:" + key }
|
|||||||
// findings is the notification an investigation ends with, finished or not.
|
// findings is the notification an investigation ends with, finished or not.
|
||||||
func findings(runID uuid.UUID, key string, res AgentResult, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
func findings(runID uuid.UUID, key string, res AgentResult, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
||||||
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
|
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
|
||||||
|
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||||
|
n.Title = "An action's outcome is unknown"
|
||||||
|
n.Body = "An action was started and its outcome can't be confirmed. It may have taken effect and is not run again; check its target. " + res.Detail
|
||||||
|
return n
|
||||||
|
}
|
||||||
if !res.Completed {
|
if !res.Completed {
|
||||||
n.Title = "Investigation stopped"
|
n.Title = "Investigation stopped"
|
||||||
n.Body = fmt.Sprintf("The investigation stopped before it finished (%s). %s", res.StopReason, res.Detail)
|
n.Body = fmt.Sprintf("The investigation stopped before it finished (%s). %s", res.StopReason, res.Detail)
|
||||||
@@ -426,7 +481,7 @@ var unsafeToolChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
|
|||||||
// toolSet turns pinned declarations into the agent's tools. Each tool calls
|
// toolSet turns pinned declarations into the agent's tools. Each tool calls
|
||||||
// its connector or MCP server by name when the model asks, so one that has
|
// its connector or MCP server by name when the model asks, so one that has
|
||||||
// since stopped fails that call rather than the run.
|
// since stopped fails that call rather than the run.
|
||||||
func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Declaration) core.ToolSet {
|
func toolSet(box Toolbox, judge Judge, rc *Context, decls []connectors.Declaration) core.ToolSet {
|
||||||
set := core.ToolSet{}
|
set := core.ToolSet{}
|
||||||
for _, d := range decls {
|
for _, d := range decls {
|
||||||
name := toolName(d.Connector, d.Name)
|
name := toolName(d.Connector, d.Name)
|
||||||
@@ -435,14 +490,14 @@ func toolSet(box Toolbox, judge Judge, runID uuid.UUID, decls []connectors.Decla
|
|||||||
Description: d.Description,
|
Description: d.Description,
|
||||||
Parameters: []byte(d.Schema),
|
Parameters: []byte(d.Schema),
|
||||||
Execute: func(ctx context.Context, input string) (string, error) {
|
Execute: func(ctx context.Context, input string) (string, error) {
|
||||||
return call(ctx, box, judge, runID, d, jsontext.Value(input))
|
return call(ctx, box, judge, rc, d, jsontext.Value(input))
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return set
|
return set
|
||||||
}
|
}
|
||||||
|
|
||||||
func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d connectors.Declaration, input jsontext.Value) (string, error) {
|
func call(ctx context.Context, box Toolbox, judge Judge, rc *Context, d connectors.Declaration, input jsontext.Value) (string, error) {
|
||||||
tool, ok := box.Tool(d.Connector)
|
tool, ok := box.Tool(d.Connector)
|
||||||
if !ok {
|
if !ok {
|
||||||
return "", fmt.Errorf("%q isn't running", d.Connector)
|
return "", fmt.Errorf("%q isn't running", d.Connector)
|
||||||
@@ -461,10 +516,15 @@ func call(ctx context.Context, box Toolbox, judge Judge, runID uuid.UUID, d conn
|
|||||||
// is proposed to a person when the investigation ends, so it comes back to
|
// is proposed to a person when the investigation ends, so it comes back to
|
||||||
// the agent as something waiting rather than a failure.
|
// the agent as something waiting rather than a failure.
|
||||||
key := core.IdempotencyKey(ctx)
|
key := core.IdempotencyKey(ctx)
|
||||||
decision, err := judge.Assess(ctx, judgement.Request{RunID: runID, Key: key, Connector: d.Connector, Action: d.Name, Input: input})
|
decision, err := judge.Assess(ctx, judgement.Request{RunID: rc.RunID(), Key: key, Connector: d.Connector, Action: d.Name, Input: input})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
if decision.Level.Runs() {
|
||||||
|
if err := begun(ctx, rc, key); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
out, err := judge.Execute(ctx, decision, input, key)
|
out, err := judge.Execute(ctx, decision, input, key)
|
||||||
if errors.Is(err, judgement.ErrHeld) {
|
if errors.Is(err, judgement.ErrHeld) {
|
||||||
return fmt.Sprintf("held at %s for a person's approval: %s. It is asked when your investigation ends, so don't propose it again.", decision.Level, findingsOf(decision)), nil
|
return fmt.Sprintf("held at %s for a person's approval: %s. It is asked when your investigation ends, so don't propose it again.", decision.Level, findingsOf(decision)), nil
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package runs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json/jsontext"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/samcharles93/ai-sdk/core"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/modules/approvals"
|
||||||
|
"github.com/samcharles93/archie-core/modules/connectors"
|
||||||
|
"github.com/samcharles93/archie-core/modules/judgement"
|
||||||
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type countingJudge struct {
|
||||||
|
level judgement.Level
|
||||||
|
executed int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *countingJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
||||||
|
return judgement.Decision{Key: req.Key, Level: j.level, Action: judgement.Action{Connector: req.Connector, Name: req.Action, Input: req.Input}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *countingJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
|
||||||
|
j.executed++
|
||||||
|
return "done", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *countingJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
|
||||||
|
j.executed++
|
||||||
|
return "done", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (j *countingJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type stubBox struct{ tool connectors.Tool }
|
||||||
|
|
||||||
|
func (stubBox) Declarations() []connectors.Declaration { return nil }
|
||||||
|
|
||||||
|
func (b stubBox) Tool(string) (connectors.Tool, bool) { return b.tool, true }
|
||||||
|
|
||||||
|
// TestActionStartedBeforeItsResultIsNeverRunAgain guards R15: a crash between
|
||||||
|
// a connector's effect and its recorded result leaves a started marker and no
|
||||||
|
// result, and the resumed run must report the outcome as unknown instead of
|
||||||
|
// running the effect a second time.
|
||||||
|
func TestActionStartedBeforeItsResultIsNeverRunAgain(t *testing.T) {
|
||||||
|
runID := uuid.MustParse("6b1f6c2e-8d5a-4c43-9f0e-1d2a3b4c5d6e")
|
||||||
|
decl := connectors.Declaration{Connector: "ops", Kind: connectors.KindAction, Name: "restart"}
|
||||||
|
marker := func(key string) []store.Step {
|
||||||
|
return []store.Step{{RunID: runID, Seq: markerBase, Name: startedPrefix + key, Outcome: store.OutcomeDone, Output: jsontext.Value("true")}}
|
||||||
|
}
|
||||||
|
agentKey := runID.String() + ":investigate:3"
|
||||||
|
approvedKey := runID.String() + ":1"
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
steps []store.Step
|
||||||
|
approved bool
|
||||||
|
wantExecuted int
|
||||||
|
wantUnknown bool
|
||||||
|
}{
|
||||||
|
{name: "agent action with no marker runs", wantExecuted: 1},
|
||||||
|
{name: "agent action with a marker is unknown", steps: marker(agentKey), wantUnknown: true},
|
||||||
|
{name: "approved action with no marker runs", approved: true, wantExecuted: 1},
|
||||||
|
{name: "approved action with a marker is unknown", steps: marker(approvedKey), approved: true, wantUnknown: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
work := &fakeWork{steps: tt.steps}
|
||||||
|
run := store.Run{ID: runID, OrgID: "org"}
|
||||||
|
rc := newContext(t.Context(), work, run, uuid.New(), tt.steps)
|
||||||
|
judge := &countingJudge{level: judgement.ActAndTell}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
if tt.approved {
|
||||||
|
d := judgement.Decision{Key: "k", Level: judgement.ProposeAndWait, Action: judgement.Action{Connector: "ops", Name: "restart"}}
|
||||||
|
_, err = runApproved(rc, judge, d, approvals.Answer{Approved: true})
|
||||||
|
} else {
|
||||||
|
ctx := core.WithIdempotencyKey(t.Context(), agentKey)
|
||||||
|
_, err = call(ctx, stubBox{}, judge, rc, decl, jsontext.Value(`{}`))
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := errors.Is(err, ErrOutcomeUnknown) && errors.Is(err, core.ErrOutcomeUnknown); got != tt.wantUnknown {
|
||||||
|
t.Fatalf("unknown outcome = %v (err %v), want %v", got, err, tt.wantUnknown)
|
||||||
|
}
|
||||||
|
if judge.executed != tt.wantExecuted {
|
||||||
|
t.Fatalf("action executed %d times, want %d", judge.executed, tt.wantExecuted)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user