981 lines
33 KiB
Go
981 lines
33 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"regexp"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/jsonschema-go/jsonschema"
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/agentloop"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
"github.com/samcharles93/ai-sdk/core"
|
|
|
|
"github.com/samcharles93/archie-core/internal/platform/prompts"
|
|
"github.com/samcharles93/archie-core/modules/approvals"
|
|
"github.com/samcharles93/archie-core/modules/connectors"
|
|
"github.com/samcharles93/archie-core/modules/deliveries"
|
|
"github.com/samcharles93/archie-core/modules/governance"
|
|
"github.com/samcharles93/archie-core/modules/judgement"
|
|
"github.com/samcharles93/archie-core/modules/memory"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// Settings provides the organisation's model configuration and escalation policy.
|
|
type Settings interface {
|
|
AgentModelForAlias(ctx context.Context, alias string) (governance.Model, error)
|
|
// AgentProvider builds the chat client for a model, resolving the
|
|
// credential it names. The model is journaled; the client is rebuilt.
|
|
AgentProvider(ctx context.Context, model governance.Model) (chat.Provider, chat.ModelInfo, error)
|
|
EscalationPath(ctx context.Context, owner string, needAuthority bool) ([]governance.PathStep, error)
|
|
RecurrenceThreshold(ctx context.Context) (int, error)
|
|
AttemptsPerRun(ctx context.Context) (int, error)
|
|
}
|
|
|
|
// Toolbox is where a workflow finds the connector lookups and actions and the
|
|
// MCP server tools it may offer an agent, and the tools that run them.
|
|
type Toolbox interface {
|
|
Declarations() []connectors.Declaration
|
|
Tool(name string) (connectors.Tool, bool)
|
|
}
|
|
|
|
// Judge decides whether an agent's action runs, and runs only what it
|
|
// decided.
|
|
type Judge interface {
|
|
Assess(ctx context.Context, req judgement.Request) (judgement.Decision, error)
|
|
Execute(ctx context.Context, d judgement.Decision, input jsontext.Value, key string) (string, error)
|
|
ExecuteApproved(ctx context.Context, d judgement.Decision, a judgement.Approval, input jsontext.Value, key string) (string, error)
|
|
ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error)
|
|
}
|
|
|
|
// Approver requests approval for actions held by judgement, escalates
|
|
// unanswered requests and expires them after the final deadline. An approval
|
|
// counts towards an increase in autonomy only when the action is about to run
|
|
// using it. If reassessment invalidates the approval, it is discarded instead.
|
|
type Approver interface {
|
|
Ask(ctx context.Context, runID uuid.UUID, key string, d judgement.Decision, path []governance.PathStep) (approvals.Request, error)
|
|
Escalate(ctx context.Context, id uuid.UUID, step int, to governance.PathStep) error
|
|
Expire(ctx context.Context, id uuid.UUID) error
|
|
Accept(ctx context.Context, id uuid.UUID) error
|
|
Discard(ctx context.Context, id uuid.UUID, because string) error
|
|
}
|
|
|
|
// Notifier tells people what a run found.
|
|
type Notifier interface {
|
|
Notify(ctx context.Context, n deliveries.Notification) error
|
|
}
|
|
|
|
// investigateBudget bounds one investigation but for its steps, which the
|
|
// organisation sets as attempts per run.
|
|
var investigateBudget = agentloop.Budget{MaxTokens: 200_000, WallClock: 10 * time.Minute}
|
|
|
|
// stepBudget is the investigation budget with the organisation's attempts per
|
|
// run. It is read, not journaled: it only caps the steps still to be taken, so
|
|
// a resumed run's recorded steps replay unchanged.
|
|
func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error) {
|
|
attempts, err := settings.AttemptsPerRun(ctx)
|
|
if err != nil {
|
|
return agentloop.Budget{}, err
|
|
}
|
|
budget := investigateBudget
|
|
budget.MaxSteps = attempts
|
|
return budget, nil
|
|
}
|
|
|
|
// Deps is what a workflow document's steps use: the organisation's settings,
|
|
// the connectors and tools an agent may call, judgement, approvals, delivery
|
|
// and memory.
|
|
type Deps struct {
|
|
Settings Settings
|
|
Toolbox Toolbox
|
|
Judge Judge
|
|
Approver Approver
|
|
Notifier Notifier
|
|
Messenger Messenger
|
|
Memory Memory
|
|
}
|
|
|
|
// Workflow is a document ready to run: the code that executes its steps for
|
|
// one run. It chooses its next step only from the document, the run's input
|
|
// and the results of earlier steps.
|
|
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
|
|
|
|
type pinnedModel struct {
|
|
model governance.Model
|
|
provider chat.Provider
|
|
info chat.ModelInfo
|
|
}
|
|
|
|
// A resumed run reuses the recorded configuration but rebuilds model
|
|
// clients and resolves tool calls against the connectors currently running.
|
|
type pinned struct {
|
|
models map[string]pinnedModel
|
|
threshold int
|
|
decls []connectors.Declaration
|
|
}
|
|
|
|
// progress is what the steps of one execution have produced so far.
|
|
type progress struct {
|
|
pin pinned
|
|
// input is the run's input as it came, and inputs the same decoded, which
|
|
// references read.
|
|
input jsontext.Value
|
|
inputs map[string]any
|
|
instructions string
|
|
results map[string]StepResult
|
|
agent *AgentResult
|
|
memory bool
|
|
remembered bool
|
|
// last is the summary of the most recent step. A run that completes every
|
|
// step uses it as its detail.
|
|
last string
|
|
outcomes []approvalOutcome
|
|
// settled is the actions already put to people, by digest, so a later
|
|
// approval step asks only about what was held since.
|
|
settled map[string]bool
|
|
review memory.Review
|
|
// stopped is an action whose outcome can't be confirmed: nothing further
|
|
// is proposed or run, and the owner is told.
|
|
stopped bool
|
|
}
|
|
|
|
// Compile returns the code that runs doc. The same steps run live and dry; a
|
|
// dry run (rc.Dry) judges every action and executes only those that read.
|
|
func (d Deps) Compile(doc Document) Workflow {
|
|
aliases, withMemory := agentUse(doc)
|
|
settleAtEnd := settlesAtEnd(doc)
|
|
return func(rc *Context, input jsontext.Value) (jsontext.Value, error) {
|
|
p := &progress{
|
|
input: input, inputs: decodeInputs(input), instructions: doc.Instructions,
|
|
results: map[string]StepResult{}, outcomes: []approvalOutcome{}, settled: map[string]bool{},
|
|
}
|
|
if len(aliases) > 0 {
|
|
if err := d.pin(rc, p, aliases, withMemory); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if parked, err := d.runSteps(rc, p, doc.Steps); err != nil {
|
|
return nil, err
|
|
} else if parked {
|
|
return p.encode()
|
|
}
|
|
// An action still held when the document ends is put to people before
|
|
// the run finishes, so nothing judgement held runs unapproved or is
|
|
// silently dropped. A run a finish step parks, declines or leaves
|
|
// waiting for approval ends as it was told to.
|
|
if settleAtEnd && !p.stopped && (rc.ending == nil || rc.ending.State == store.StateSucceeded) {
|
|
if _, err := d.settle(rc, p); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if rc.ending == nil {
|
|
rc.End(store.StateSucceeded, p.last)
|
|
}
|
|
return p.result()
|
|
}
|
|
}
|
|
|
|
// runSteps runs the steps in order until one ends the run. It reports whether
|
|
// a step failed and parked the run, for an operator to retry from that step;
|
|
// a step that continues on failure does not.
|
|
func (d Deps) runSteps(rc *Context, p *progress, steps []Step) (parked bool, err error) {
|
|
for i, s := range steps {
|
|
if p.stopped && s.Type != typeNotify {
|
|
continue
|
|
}
|
|
name := s.name(i)
|
|
if err := rc.Enter(rc.Context(), name); err != nil {
|
|
return false, err
|
|
}
|
|
if err := d.runStep(rc, p, s, i); err != nil {
|
|
if !stepFailed(err) {
|
|
return false, err
|
|
}
|
|
rc.End(store.StateParked, fmt.Sprintf("step %s: %v", name, err))
|
|
return true, nil
|
|
}
|
|
if rc.ending != nil {
|
|
break
|
|
}
|
|
}
|
|
return false, nil
|
|
}
|
|
|
|
// runStep resolves settings from the run's inputs and earlier results, and
|
|
// saves the result under the step ID if the step runs.
|
|
func (d Deps) runStep(rc *Context, p *progress, s Step, index int) error {
|
|
name := s.name(index)
|
|
values := scope(p.inputs, p.results)
|
|
if s.When != "" {
|
|
// The condition reads only the inputs and recorded results, so a replay
|
|
// decides the same way; the skip is recorded for the trail.
|
|
if cond, err := parseCondition(s.When); err == nil && !cond.holds(values) {
|
|
_, err := rc.Step("skip:"+name, func(context.Context, string) (string, error) {
|
|
return s.When, nil
|
|
})
|
|
return err
|
|
}
|
|
}
|
|
res, err := stepTypes[s.Type].run(d, rc, p, name, renderSettings(s.Settings, values))
|
|
if err != nil {
|
|
if s.OnFailure != onFailureContinue || !stepFailed(err) {
|
|
return err
|
|
}
|
|
res = StepResult{Summary: err.Error()}
|
|
}
|
|
if s.ID != "" {
|
|
p.results[s.ID] = res
|
|
}
|
|
p.last = res.Summary
|
|
return nil
|
|
}
|
|
|
|
// stepFailed reports whether err is the step's own failure, which on_failure
|
|
// may continue past, rather than the run being suspended, cancelled, replayed
|
|
// differently or losing its lease.
|
|
func stepFailed(err error) bool {
|
|
for _, stop := range []error{errSuspended, store.ErrLeaseLost, context.Canceled, ErrCancelled, ErrNondeterministic} {
|
|
if errors.Is(err, stop) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func agentUse(doc Document) (aliases []string, withMemory bool) {
|
|
for _, s := range doc.Steps {
|
|
if s.Type != typeAgent {
|
|
continue
|
|
}
|
|
var settings agentSettings
|
|
if decodeSettings(s.Settings, &settings) == nil {
|
|
alias := modelAlias(settings.Model)
|
|
if !slices.Contains(aliases, alias) {
|
|
aliases = append(aliases, alias)
|
|
}
|
|
withMemory = withMemory || settings.Memory
|
|
}
|
|
}
|
|
return aliases, withMemory
|
|
}
|
|
|
|
func modelAlias(alias string) string {
|
|
if alias == "default" {
|
|
return ""
|
|
}
|
|
return alias
|
|
}
|
|
|
|
// settlesAtEnd reports whether the run must put held actions to people after
|
|
// its last step. It does when a step that can propose an action comes after the
|
|
// last approval step, or when that approval step is conditional: a skipped
|
|
// approval leaves everything held before it unsettled.
|
|
func settlesAtEnd(doc Document) bool {
|
|
settled := true
|
|
for _, s := range doc.Steps {
|
|
switch {
|
|
case s.Type == typeApproval && s.When == "" && s.OnFailure != onFailureContinue:
|
|
settled = true
|
|
case s.Type == typeApproval:
|
|
// A conditional approval may not run, and one that continues past
|
|
// its own failure may not settle what it asked, so anything held
|
|
// before it stays unsettled unless a later one settles it.
|
|
case holds(s.Type):
|
|
settled = false
|
|
}
|
|
}
|
|
return !settled
|
|
}
|
|
|
|
// decodeInputs reads the run's input as the object references resolve inputs
|
|
// against. Input that isn't an object has no named inputs.
|
|
func decodeInputs(input jsontext.Value) map[string]any {
|
|
var out map[string]any
|
|
if len(input) == 0 || json.Unmarshal(input, &out) != nil || out == nil {
|
|
return map[string]any{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (d Deps) pin(rc *Context, p *progress, aliases []string, withMemory bool) error {
|
|
models := make(map[string]pinnedModel, len(aliases))
|
|
for _, alias := range aliases {
|
|
step := "agent-model"
|
|
if alias != "" {
|
|
step += ":" + alias
|
|
}
|
|
model, err := rc.Step(step, func(ctx context.Context, _ string) (governance.Model, error) {
|
|
return d.Settings.AgentModelForAlias(ctx, alias)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
models[alias] = pinnedModel{model: model}
|
|
}
|
|
if withMemory {
|
|
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
|
return d.Settings.RecurrenceThreshold(ctx)
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.pin.threshold = threshold
|
|
}
|
|
decls, err := rc.Step("agent-tools", func(context.Context, string) ([]connectors.Declaration, error) {
|
|
return d.Toolbox.Declarations(), nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, alias := range aliases {
|
|
m := models[alias]
|
|
if m.provider, m.info, err = d.Settings.AgentProvider(rc.Context(), m.model); err != nil {
|
|
return err
|
|
}
|
|
models[alias] = m
|
|
}
|
|
p.pin.models, p.pin.decls = models, decls
|
|
return nil
|
|
}
|
|
|
|
// agentSystem is an agent step's system prompt: how Archie works, then the
|
|
// workflow's instructions, then the step's mission.
|
|
func agentSystem(instructions, mission string) string {
|
|
parts := []string{prompts.Agent()}
|
|
if strings.TrimSpace(instructions) != "" {
|
|
parts = append(parts, instructions)
|
|
}
|
|
return strings.Join(append(parts, mission), "\n\n")
|
|
}
|
|
|
|
// runAgent evaluates each action when the agent calls it. Actions that
|
|
// require human approval are deferred to an approval step.
|
|
func runAgent(d Deps, rc *Context, p *progress, name string, s agentSettings) (StepResult, error) {
|
|
budget, err := stepBudget(rc.Context(), d.Settings)
|
|
if err != nil {
|
|
return StepResult{}, err
|
|
}
|
|
decls := p.pin.decls
|
|
if s.ReadOnly {
|
|
decls = readOnly(decls)
|
|
}
|
|
var (
|
|
resultRaw []byte
|
|
resultSchema *jsonschema.Resolved
|
|
)
|
|
if s.Result != nil {
|
|
if resultRaw, resultSchema, err = s.resultSchema(); err != nil {
|
|
return StepResult{}, fmt.Errorf("step %q result: %w", name, err)
|
|
}
|
|
}
|
|
tools := func() core.ToolSet {
|
|
set := toolSet(d.Toolbox, d.Judge, rc, decls)
|
|
if s.Memory {
|
|
set = withMemory(set, d.Memory, rc.RunID())
|
|
}
|
|
if resultSchema != nil {
|
|
set[resultTool] = newResultTool(resultRaw, resultSchema)
|
|
}
|
|
return set
|
|
}
|
|
model := p.pin.models[modelAlias(s.Model)]
|
|
base := AgentRequest{
|
|
Provider: model.provider, Model: model.model.Model, ModelInfo: model.info,
|
|
System: agentSystem(p.instructions, s.system()), Tools: tools(),
|
|
}
|
|
first := base
|
|
first.Name, first.Prompt, first.Budget = name, string(p.input), budget
|
|
if resultSchema != nil {
|
|
first.Capture = resultTool
|
|
}
|
|
res, err := rc.Agent(first)
|
|
if err != nil {
|
|
return StepResult{}, err
|
|
}
|
|
returnedResult := res.Returned
|
|
followUp := base
|
|
followUp.Tools = tools()
|
|
if res, err = followThrough(rc, d.Judge, decls, p.input, res, followUp); err != nil {
|
|
return StepResult{}, err
|
|
}
|
|
p.agent = &res
|
|
if res.StopReason == agentloop.StopOutcomeUnknown {
|
|
p.stopped = true
|
|
return StepResult{Summary: res.Answer}, nil
|
|
}
|
|
if s.Memory {
|
|
p.memory = true
|
|
if !rc.Dry() {
|
|
if err := recordCause(rc, d.Memory, p.input, res, base); err != nil {
|
|
return StepResult{}, err
|
|
}
|
|
}
|
|
}
|
|
return agentResult(name, res, returnedResult, resultSchema)
|
|
}
|
|
|
|
func agentResult(name string, res AgentResult, returned jsontext.Value, schema *jsonschema.Resolved) (StepResult, error) {
|
|
out := StepResult{Summary: res.Answer}
|
|
if schema == nil || !res.Completed {
|
|
return out, nil
|
|
}
|
|
if len(returned) == 0 {
|
|
return StepResult{}, fmt.Errorf("step %q: the agent finished without returning its result", name)
|
|
}
|
|
result, err := decodeResult(returned, schema)
|
|
if err != nil {
|
|
return StepResult{}, fmt.Errorf("step %q: %w", name, err)
|
|
}
|
|
out.Result = result
|
|
return out, nil
|
|
}
|
|
|
|
// result is the run's output: the last agent's findings, or what became of the
|
|
// actions when no agent ran. An agent that did not complete fails the run once
|
|
// the owner has been told.
|
|
func (p *progress) result() (jsontext.Value, error) {
|
|
out, err := p.encode()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if p.agent != nil && !p.agent.Completed {
|
|
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// encode returns the run's findings so far as the result that the run records.
|
|
func (p *progress) encode() (jsontext.Value, error) {
|
|
var v any = p.outcomes
|
|
if p.agent != nil {
|
|
v = *p.agent
|
|
}
|
|
out, err := json.Marshal(v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("encode result: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// begun records that an action is about to run. It fails with an unknown
|
|
// outcome when an earlier attempt already started it, so the action is not run
|
|
// again.
|
|
func begun(ctx context.Context, rc *Context, key string) error {
|
|
started, err := rc.Begin(ctx, key)
|
|
switch {
|
|
case err != nil:
|
|
return err
|
|
case started:
|
|
return fmt.Errorf("%w: %w: an earlier attempt started it and did not record how it ended", ErrOutcomeUnknown, core.ErrOutcomeUnknown)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// followThroughBudget bounds the turn that has the agent call what it only
|
|
// described.
|
|
var followThroughBudget = agentloop.Budget{MaxSteps: 8, MaxTokens: 60_000, WallClock: 5 * time.Minute}
|
|
|
|
// followThrough gives the agent one more turn when its findings name an action
|
|
// it never called: a small model often writes the call out instead of making
|
|
// it, and an action that was only described can't be judged or approved.
|
|
func followThrough(rc *Context, judge Judge, decls []connectors.Declaration, input jsontext.Value, res AgentResult, req AgentRequest) (AgentResult, error) {
|
|
if !res.Completed {
|
|
return res, nil
|
|
}
|
|
made, err := rc.Step("decisions-made", func(ctx context.Context, _ string) ([]string, error) {
|
|
decisions, err := judge.ForRun(ctx, rc.RunID())
|
|
names := make([]string, 0, len(decisions))
|
|
for _, d := range decisions {
|
|
names = append(names, toolName(d.Action.Connector, d.Action.Name))
|
|
}
|
|
return names, err
|
|
})
|
|
if err != nil {
|
|
return res, err
|
|
}
|
|
missed := uncalled(res.Answer, decls, made)
|
|
if len(missed) == 0 {
|
|
return res, nil
|
|
}
|
|
// The prompt is part of the journaled request digest, so its key order must be the same on replay.
|
|
prompt, err := json.Marshal(map[string]any{"event": input, "findings": res.Answer, "described_but_not_called": missed}, json.Deterministic(true))
|
|
if err != nil {
|
|
return res, err
|
|
}
|
|
req.Name, req.System, req.Prompt, req.Budget = "follow-through", req.System+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
|
again, err := rc.Agent(req)
|
|
if err == nil && again.StopReason == agentloop.StopOutcomeUnknown {
|
|
return again, nil
|
|
}
|
|
if err != nil || !again.Completed {
|
|
// The first findings stand when the extra turn doesn't finish.
|
|
return res, err
|
|
}
|
|
return again, nil
|
|
}
|
|
|
|
// uncalled lists the actions an answer names that no decision in the run
|
|
// records.
|
|
func uncalled(answer string, decls []connectors.Declaration, made []string) []string {
|
|
var out []string
|
|
for _, d := range decls {
|
|
name := toolName(d.Connector, d.Name)
|
|
if d.Kind == connectors.KindAction && strings.Contains(answer, name) && !slices.Contains(made, name) {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func toolName(connector, name string) string {
|
|
return unsafeToolChars.ReplaceAllString(connector+"_"+name, "_")
|
|
}
|
|
|
|
// approvalOutcome is what became of one action the agent proposed.
|
|
type approvalOutcome struct {
|
|
Action string `json:"action"`
|
|
Target string `json:"target"`
|
|
RequestID uuid.UUID `json:"request_id"`
|
|
State string `json:"state"`
|
|
By string `json:"by,omitzero"`
|
|
digest string
|
|
Result string `json:"result,omitzero"`
|
|
Error string `json:"error,omitzero"`
|
|
// Unknown is an action that was started but can't be confirmed: it may
|
|
// have taken effect, and it is never started again.
|
|
Unknown bool `json:"unknown,omitzero"`
|
|
}
|
|
|
|
// settleProposals puts every action judgement held and nobody settled yet to a
|
|
// person, one at a time, and runs the ones they allow. Each ask, wait and run is its own step, so a
|
|
// resumed run picks up where it waited. A proposal nobody answers before the
|
|
// last escalation step runs out is settled as expired and nothing runs.
|
|
func settleProposals(rc *Context, judge Judge, ask Approver, settings Settings, settled map[string]bool) ([]approvalOutcome, error) {
|
|
held, err := heldSince(rc, judge, settled)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]approvalOutcome, 0, len(held))
|
|
for _, d := range held {
|
|
outcome, err := settleOne(rc, judge, ask, settings, d)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, outcome)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// heldSince journals all actions held for approval before filtering out
|
|
// settled ones so that replay uses the same list.
|
|
func heldSince(rc *Context, judge Judge, settled map[string]bool) ([]judgement.Decision, error) {
|
|
held, err := rc.Step("proposals", func(ctx context.Context, _ string) ([]judgement.Decision, error) {
|
|
decisions, err := judge.ForRun(ctx, rc.RunID())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return heldByJudgement(decisions), nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return slices.DeleteFunc(held, func(d judgement.Decision) bool { return settled[d.Action.Digest] }), nil
|
|
}
|
|
|
|
// maxReapprovals limits the number of new requests for one action after
|
|
// reassessment invalidates an approval and the action still requires approval.
|
|
// The initial request and escalation within a request do not count.
|
|
const maxReapprovals = 2
|
|
|
|
// settleOne gets one action approved and runs it. Before it runs it is judged
|
|
// again; an approval whose action has since changed is discarded and the
|
|
// changed action is put to people as a new request.
|
|
func settleOne(rc *Context, judge Judge, ask Approver, settings Settings, d judgement.Decision) (approvalOutcome, error) {
|
|
out := approvalOutcome{digest: d.Action.Digest}
|
|
for round := 0; ; round++ {
|
|
out.Action, out.Target = d.Action.Connector+" "+d.Action.Name, d.Action.Target.Kind+":"+d.Action.Target.ID
|
|
request, answer, err := askThroughPath(rc, ask, settings, d)
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
out.RequestID, out.State = request.ID, request.State
|
|
if request.State != approvals.StateApproved {
|
|
if request.State == approvals.StateDenied {
|
|
out.By = answer.By
|
|
}
|
|
return out, nil
|
|
}
|
|
out.By = answer.By
|
|
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
|
|
return judge.Assess(ctx, rc.request(key, d.Action.Connector, d.Action.Name, d.Action.Input))
|
|
})
|
|
if err != nil {
|
|
return out, err
|
|
}
|
|
next, why := afterApproval(d, fresh)
|
|
switch next {
|
|
case reaskPerson, refuseAction:
|
|
if _, err := rc.Step("discard:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
|
|
return true, ask.Discard(ctx, request.ID, why)
|
|
}); err != nil {
|
|
return out, err
|
|
}
|
|
out.State = approvals.StateDiscarded
|
|
if next == refuseAction || round == maxReapprovals {
|
|
out.Error = why
|
|
return out, nil
|
|
}
|
|
d = fresh
|
|
continue
|
|
case runFresh:
|
|
// The approval isn't needed and isn't counted.
|
|
out.Result, out.Error, out.Unknown = runJudged(rc, judge, fresh, nil)
|
|
default:
|
|
if _, err := rc.Step("count:"+d.Key, func(ctx context.Context, _ string) (bool, error) {
|
|
return true, ask.Accept(ctx, request.ID)
|
|
}); err != nil {
|
|
return out, err
|
|
}
|
|
out.Result, out.Error, out.Unknown = runJudged(rc, judge, d, &answer)
|
|
}
|
|
out.State = approvals.StateApproved
|
|
return out, nil
|
|
}
|
|
}
|
|
|
|
// askThroughPath puts d to the first step of its escalation path and waits.
|
|
// The path is pinned when it is first asked, so a request that waits keeps the
|
|
// deadlines it started with whatever the settings become meanwhile. It returns
|
|
// the request as it stood when it settled, and the answer if there was one.
|
|
func askThroughPath(rc *Context, ask Approver, settings Settings, d judgement.Decision) (approvals.Request, approvals.Answer, error) {
|
|
path, err := rc.Step("path:"+d.Key, func(ctx context.Context, _ string) ([]governance.PathStep, error) {
|
|
return settings.EscalationPath(ctx, d.Owner(), d.Level == judgement.Escalate)
|
|
})
|
|
if err != nil {
|
|
return approvals.Request{}, approvals.Answer{}, err
|
|
}
|
|
request, err := rc.Step("ask:"+d.Key, func(ctx context.Context, _ string) (approvals.Request, error) {
|
|
return ask.Ask(ctx, rc.RunID(), waitKey(d.Key), d, path)
|
|
})
|
|
if err != nil {
|
|
return request, approvals.Answer{}, err
|
|
}
|
|
if request.State != approvals.StatePending && request.State != approvals.StateApproved {
|
|
return request, approvals.Answer{}, nil
|
|
}
|
|
// A request already answered when it was asked again still has its answer
|
|
// waiting under the run's key, so the wait returns it at once.
|
|
answer, expired, err := waitThroughPath(rc, ask, d.Key, request, path)
|
|
switch {
|
|
case err != nil:
|
|
return request, answer, err
|
|
case expired:
|
|
request.State = approvals.StateExpired
|
|
case answer.Approved:
|
|
request.State = approvals.StateApproved
|
|
default:
|
|
request.State = approvals.StateDenied
|
|
}
|
|
return request, answer, nil
|
|
}
|
|
|
|
// runJudged runs the action judged as d, on approval when there is one, and
|
|
// returns what came of it. The action's own failure is part of what the answer
|
|
// led to, and its step has recorded it, so the investigation still reports.
|
|
func runJudged(rc *Context, judge Judge, d judgement.Decision, approval *approvals.Answer) (result, failure string, unknown bool) {
|
|
var runErr error
|
|
if approval == nil {
|
|
result, runErr = runUnattended(rc, judge, d)
|
|
} else {
|
|
result, runErr = runApproved(rc, judge, d, *approval)
|
|
}
|
|
return result, errText(runErr), errors.Is(runErr, ErrOutcomeUnknown)
|
|
}
|
|
|
|
// unknownOutcome records a connector's unconfirmed action as a step whose
|
|
// outcome is unknown, so a resumed run reports it and never runs it again.
|
|
func unknownOutcome(err error) error {
|
|
if errors.Is(err, connectors.ErrOutcomeUnknown) {
|
|
return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// errText is a failed action's reason, or nothing when it ran.
|
|
func errText(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
// waitThroughPath waits for an answer through each step of path in turn. A step
|
|
// that runs out with none passes the request to the next person; the last one
|
|
// settles it as expired and the caller runs nothing. Every wait is a durable
|
|
// run wait, so a restart resumes where each deadline stands.
|
|
func waitThroughPath(rc *Context, ask Approver, key string, request approvals.Request, path []governance.PathStep) (approvals.Answer, bool, error) {
|
|
for i, step := range path {
|
|
if i > 0 {
|
|
if _, err := rc.Step("escalate:"+key+":"+strconv.Itoa(i), func(ctx context.Context, _ string) (bool, error) {
|
|
return true, ask.Escalate(ctx, request.ID, i, step)
|
|
}); err != nil {
|
|
return approvals.Answer{}, false, err
|
|
}
|
|
}
|
|
payload, err := rc.Wait(waitKey(key), step.Deadline())
|
|
if errors.Is(err, ErrWaitTimedOut) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return approvals.Answer{}, false, err
|
|
}
|
|
var answer approvals.Answer
|
|
if err := json.Unmarshal(payload, &answer); err != nil {
|
|
return approvals.Answer{}, false, fmt.Errorf("decode approval answer: %w", err)
|
|
}
|
|
return answer, false, nil
|
|
}
|
|
if _, err := rc.Step("expire:"+key, func(ctx context.Context, _ string) (bool, error) {
|
|
return true, ask.Expire(ctx, request.ID)
|
|
}); err != nil {
|
|
return approvals.Answer{}, true, err
|
|
}
|
|
return approvals.Answer{}, true, nil
|
|
}
|
|
|
|
// What an approval allows once the action is judged again.
|
|
type afterwards int
|
|
|
|
const (
|
|
// runApproval runs the action that was approved.
|
|
runApproval afterwards = iota
|
|
// runFresh runs the action on its new judgement, which lets it run without
|
|
// anyone's approval.
|
|
runFresh
|
|
// reaskPerson discards the approval and puts the changed action to people.
|
|
reaskPerson
|
|
// refuseAction discards the approval: the action is now refused.
|
|
refuseAction
|
|
)
|
|
|
|
// afterApproval checks whether an approval is still valid after reassessment.
|
|
// If the action still requires approval, a stricter verdict or a change to the
|
|
// resolved target or effect requires a new request.
|
|
func afterApproval(approved, fresh judgement.Decision) (afterwards, string) {
|
|
switch {
|
|
case fresh.Level == judgement.Never:
|
|
return refuseAction, "refused after approval: " + fresh.Reason
|
|
case fresh.Level.Runs():
|
|
return runFresh, ""
|
|
case fresh.Action.Target != approved.Action.Target:
|
|
return reaskPerson, "the target changed since it was approved"
|
|
case fresh.Action.Effect != approved.Action.Effect:
|
|
return reaskPerson, "the effect changed since it was approved"
|
|
case fresh.Level > approved.Level:
|
|
return reaskPerson, "judgement is stricter than when the action was approved: now " + fresh.Level.String()
|
|
}
|
|
return runApproval, ""
|
|
}
|
|
|
|
// runUnattended runs an action its fresh judgement lets run without approval,
|
|
// once.
|
|
func runUnattended(rc *Context, judge Judge, d judgement.Decision) (string, error) {
|
|
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
|
|
if err := begun(ctx, rc, key); err != nil {
|
|
return "", err
|
|
}
|
|
out, err := judge.Execute(ctx, d, d.Action.Input, key)
|
|
return out, unknownOutcome(err)
|
|
})
|
|
}
|
|
|
|
// runApproved runs the action a person approved, once. judgement runs only the
|
|
// action that was judged, whatever input comes with it.
|
|
func runApproved(rc *Context, judge Judge, d judgement.Decision, answer approvals.Answer) (string, error) {
|
|
return rc.Step("run:"+d.Key, func(ctx context.Context, key string) (string, error) {
|
|
if err := begun(ctx, rc, key); err != nil {
|
|
return "", err
|
|
}
|
|
out, err := judge.ExecuteApproved(ctx, d, answer.Approval(d), d.Action.Input, key)
|
|
return out, unknownOutcome(err)
|
|
})
|
|
}
|
|
|
|
// heldByJudgement is the actions an agent proposed that need a person before
|
|
// they run, once each: an agent that asks for the same action twice is asked
|
|
// about it once.
|
|
func heldByJudgement(decisions []judgement.Decision) []judgement.Decision {
|
|
seen := make(map[string]bool, len(decisions))
|
|
out := make([]judgement.Decision, 0, len(decisions))
|
|
for _, d := range decisions {
|
|
if d.Level < judgement.ProposeAndWait || d.Level == judgement.Never || seen[d.Action.Digest] {
|
|
continue
|
|
}
|
|
seen[d.Action.Digest] = true
|
|
out = append(out, d)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// waitKey is the run key an approval answer wakes.
|
|
func waitKey(key string) string { return "approval:" + key }
|
|
|
|
// findings is the notification a run ends with, finished or not.
|
|
func findings(runID uuid.UUID, key string, agent *AgentResult, title string, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
|
if agent == nil {
|
|
if title == "" {
|
|
title = "Run finished"
|
|
}
|
|
return deliveries.Notification{RunID: runID, Key: key, Title: title, Body: outcomeLines(outcomes)}
|
|
}
|
|
res := *agent
|
|
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
|
|
if title != "" {
|
|
n.Title = title
|
|
}
|
|
if res.StopReason == agentloop.StopOutcomeUnknown {
|
|
n.Title = "An action's outcome is unknown"
|
|
n.Body = "An action was started and its outcome can't be confirmed. It may have taken effect and is not run again; check its target. " + res.Detail
|
|
return n
|
|
}
|
|
if !res.Completed {
|
|
n.Title = "Investigation stopped"
|
|
n.Body = fmt.Sprintf("The investigation stopped before it finished (%s). %s", res.StopReason, res.Detail)
|
|
}
|
|
// The record says plainly when nothing was proposed, whatever the findings
|
|
// then claim.
|
|
n.Body += "\n\n" + outcomeLines(outcomes)
|
|
if note := recurrenceLines(review); note != "" {
|
|
n.Body += "\n\n" + note
|
|
}
|
|
return n
|
|
}
|
|
|
|
// outcomeLines says what became of each action the investigation proposed, and
|
|
// that none was when that is the case.
|
|
func outcomeLines(outcomes []approvalOutcome) string {
|
|
if len(outcomes) == 0 {
|
|
return "Actions: none proposed."
|
|
}
|
|
lines := make([]string, 0, len(outcomes))
|
|
for _, o := range outcomes {
|
|
line := fmt.Sprintf("- %s on %s: %s", o.Action, o.Target, o.State)
|
|
switch {
|
|
case o.Unknown:
|
|
line += ", it was started but its outcome is unknown: " + o.Error
|
|
case o.Error != "":
|
|
line += ", it did not run: " + o.Error
|
|
case o.Result != "":
|
|
line += ", it ran: " + o.Result
|
|
case o.State == approvals.StateApproved:
|
|
line += ", it ran"
|
|
}
|
|
if o.By != "" {
|
|
line += " (" + o.By + ")"
|
|
}
|
|
lines = append(lines, line)
|
|
}
|
|
return "Actions:\n" + strings.Join(lines, "\n")
|
|
}
|
|
|
|
// withMemory adds the lookup of a target's earlier incidents to the agent's
|
|
// tools.
|
|
func withMemory(set core.ToolSet, mem Memory, runID uuid.UUID) core.ToolSet {
|
|
tool := pastIncidentsTool(mem, runID)
|
|
set[tool.Name] = tool
|
|
return set
|
|
}
|
|
|
|
// unsafeToolChars are those a model API refuses in a tool name.
|
|
var unsafeToolChars = regexp.MustCompile(`[^a-zA-Z0-9_-]`)
|
|
|
|
// toolSet turns pinned declarations into the agent's tools. Each tool calls
|
|
// its connector or MCP server by name when the model asks, so one that has
|
|
// since stopped fails that call rather than the run.
|
|
func toolSet(box Toolbox, judge Judge, rc *Context, decls []connectors.Declaration) core.ToolSet {
|
|
set := core.ToolSet{}
|
|
for _, d := range decls {
|
|
name := toolName(d.Connector, d.Name)
|
|
set[name] = &core.Tool{
|
|
Name: name,
|
|
Description: d.Description,
|
|
Parameters: []byte(d.Schema),
|
|
Execute: func(ctx context.Context, input string) (string, error) {
|
|
return call(ctx, box, judge, rc, d, jsontext.Value(input))
|
|
},
|
|
}
|
|
}
|
|
return set
|
|
}
|
|
|
|
func call(ctx context.Context, box Toolbox, judge Judge, rc *Context, d connectors.Declaration, input jsontext.Value) (string, error) {
|
|
tool, ok := box.Tool(d.Connector)
|
|
if !ok {
|
|
return "", fmt.Errorf("%q isn't running", d.Connector)
|
|
}
|
|
if d.Kind == connectors.KindLookup {
|
|
out, found, err := tool.Lookup(ctx, d.Name, input)
|
|
switch {
|
|
case err != nil:
|
|
return "", err
|
|
case !found:
|
|
return "not found", nil
|
|
}
|
|
return out, nil
|
|
}
|
|
// An action is judged first and runs only as decided. One judgement holds
|
|
// is proposed to a person when the investigation ends, so it comes back to
|
|
// the agent as something waiting rather than a failure.
|
|
key := core.IdempotencyKey(ctx)
|
|
decision, err := judge.Assess(ctx, rc.request(key, d.Connector, d.Name, input))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if decision.Level.Runs() && !rc.mayExecute(decision) {
|
|
return fmt.Sprintf("dry run: this would run now and was not executed: %s. Carry on as if it had succeeded.", decision.Action.Effect.Summary), nil
|
|
}
|
|
if decision.Level.Runs() {
|
|
if err := begun(ctx, rc, key); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
out, err := judge.Execute(ctx, decision, input, key)
|
|
if errors.Is(err, judgement.ErrHeld) {
|
|
return fmt.Sprintf("held at %s for approval: %s. It is asked when your investigation ends, so don't propose it again.", decision.Level, findingsOf(decision)), nil
|
|
}
|
|
if errors.Is(err, connectors.ErrOutcomeUnknown) {
|
|
return "", fmt.Errorf("%w: %w", core.ErrOutcomeUnknown, err)
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// findingsOf lists what held a decision back, for the agent.
|
|
func findingsOf(d judgement.Decision) string {
|
|
var parts []string
|
|
for _, v := range d.Verdicts {
|
|
if v.Advisory || v.Ceiling != d.Level {
|
|
continue
|
|
}
|
|
for _, f := range v.Findings {
|
|
parts = append(parts, v.Guard+": "+f.Text)
|
|
}
|
|
if v.Failed != "" {
|
|
parts = append(parts, v.Guard+" couldn't judge: "+v.Failed)
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
return "no guard allowed it to run unattended"
|
|
}
|
|
return strings.Join(parts, "; ")
|
|
}
|