refactor(runs): a workflow is a stored document of agent, action, wait-for-approval, notify and sleep steps, and investigate ships as one; dry mode judges without executing (#2008)
This commit is contained in:
12 files changed
+1113
-190
No files matched your search
@@ -20,6 +20,14 @@ func Investigate() string { return investigate }
|
||||
// action it didn't call.
|
||||
func FollowThrough() string { return followThrough }
|
||||
|
||||
// Agent returns the system prompt an agent step of a workflow document names.
|
||||
func Agent(name string) (string, bool) {
|
||||
if name == "investigate" {
|
||||
return investigate, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// RecordCause is the system prompt of the turn that files an investigation's
|
||||
// cause in Archie's memory.
|
||||
func RecordCause() string { return recordCause }
|
||||
@@ -118,7 +118,7 @@ func TestAgentResumesFromItsRecordedCalls(t *testing.T) {
|
||||
func execute(ctx context.Context, t *testing.T, work *fakeWork, model chat.Provider, toolRuns *int, duringTool func()) {
|
||||
t.Helper()
|
||||
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
||||
engine.Register("agent", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
if _, err := rc.Step("before", func(context.Context, string) (bool, error) { return true, nil }); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -45,6 +45,8 @@ type Context struct {
|
||||
// are numbered from nextMarker.
|
||||
started map[string]bool
|
||||
nextMarker int
|
||||
// overlay is what a dry run carries instead of a stored version.
|
||||
overlay *dryOverlay
|
||||
}
|
||||
|
||||
// markerBase is where started markers are numbered, far above any step a
|
||||
@@ -64,7 +66,11 @@ func newContext(ctx context.Context, work workerStore, run store.Run, token uuid
|
||||
nextMarker = max(nextMarker, s.Seq+1)
|
||||
}
|
||||
}
|
||||
return &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
|
||||
rc := &Context{ctx: ctx, work: work, run: run, token: token, saved: saved, started: started, nextMarker: nextMarker}
|
||||
if run.Dry {
|
||||
rc.overlay, _ = readOverlay(run.Overlay)
|
||||
}
|
||||
return rc
|
||||
}
|
||||
|
||||
// Begin durably records that the effect behind key is about to happen, before
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed" // the starter workflow documents
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/prompts"
|
||||
)
|
||||
|
||||
//go:embed workflows/investigate.yaml
|
||||
var investigateDocument string
|
||||
|
||||
// InvestigateDocument is the workflow document that has an agent look into the
|
||||
// event that started the run, put what it proposes to people, and report.
|
||||
func InvestigateDocument() string { return investigateDocument }
|
||||
|
||||
// Document is a workflow as data: an ordered list of steps the engine
|
||||
// executes. It is stored, diffed and validated as text.
|
||||
type Document struct {
|
||||
Steps []Step `yaml:"steps"`
|
||||
}
|
||||
|
||||
// Step is one step of a document. Exactly one of the kinds is set.
|
||||
type Step struct {
|
||||
ID string `yaml:"id"`
|
||||
Agent *AgentStep `yaml:"agent"`
|
||||
Action *ActionStep `yaml:"action"`
|
||||
WaitForApproval *ApprovalStep `yaml:"wait_for_approval"`
|
||||
Notify *NotifyStep `yaml:"notify"`
|
||||
Sleep *SleepStep `yaml:"sleep"`
|
||||
}
|
||||
|
||||
// AgentStep runs a tool-using agent on the event. Prompt names one of the
|
||||
// binary's prompts. Memory gives the agent the lookup of a target's earlier
|
||||
// incidents and files what the run found and did once it ends.
|
||||
type AgentStep struct {
|
||||
Prompt string `yaml:"prompt"`
|
||||
Memory bool `yaml:"memory"`
|
||||
}
|
||||
|
||||
// ActionStep judges one connector action and runs it if judgement lets it run
|
||||
// unattended. A string input value written {{event.path}} is replaced by the
|
||||
// value at that path of the event.
|
||||
type ActionStep struct {
|
||||
Connector string `yaml:"connector"`
|
||||
Name string `yaml:"name"`
|
||||
Input map[string]any `yaml:"input"`
|
||||
}
|
||||
|
||||
// ApprovalStep puts every action judgement held so far to people, and runs the
|
||||
// ones they allow.
|
||||
type ApprovalStep struct{}
|
||||
|
||||
// NotifyStep tells people what the run found. Title replaces the default.
|
||||
type NotifyStep struct {
|
||||
Title string `yaml:"title"`
|
||||
}
|
||||
|
||||
// SleepStep suspends the run for a duration such as "30s".
|
||||
type SleepStep struct {
|
||||
For string `yaml:"for"`
|
||||
}
|
||||
|
||||
// maxSleep bounds a sleep step.
|
||||
const maxSleep = 24 * time.Hour
|
||||
|
||||
var stepID = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`)
|
||||
|
||||
// Problem is one thing wrong with a document, at the path of the field.
|
||||
type Problem struct {
|
||||
Path string `json:"path"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (p Problem) String() string { return p.Path + ": " + p.Message }
|
||||
|
||||
// InvalidError is the problems found in a document.
|
||||
type InvalidError struct {
|
||||
Problems []Problem `json:"problems"`
|
||||
}
|
||||
|
||||
func (e *InvalidError) Error() string {
|
||||
parts := make([]string, len(e.Problems))
|
||||
for i, p := range e.Problems {
|
||||
parts[i] = p.String()
|
||||
}
|
||||
return "invalid workflow: " + strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// ParseDocument reads a document and validates it. Fields it doesn't know are
|
||||
// an error, so a misspelt option is never silently ignored.
|
||||
func ParseDocument(body string) (Document, error) {
|
||||
var doc Document
|
||||
dec := yaml.NewDecoder(strings.NewReader(body))
|
||||
dec.KnownFields(true)
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return Document{}, &InvalidError{Problems: []Problem{{Path: "steps", Message: "a workflow needs steps"}}}
|
||||
}
|
||||
return Document{}, &InvalidError{Problems: yamlProblems(err)}
|
||||
}
|
||||
if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
|
||||
return Document{}, &InvalidError{Problems: []Problem{{Path: "document", Message: "one workflow per document"}}}
|
||||
}
|
||||
if problems := doc.validate(); len(problems) > 0 {
|
||||
return Document{}, &InvalidError{Problems: problems}
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
var unknownField = regexp.MustCompile(`^(line \d+): field (\S+) not found`)
|
||||
|
||||
// yamlProblems turns the decoder's errors into one problem each, an unknown
|
||||
// field at its line.
|
||||
func yamlProblems(err error) []Problem {
|
||||
var typed *yaml.TypeError
|
||||
if !errors.As(err, &typed) {
|
||||
return []Problem{{Path: "document", Message: strings.TrimPrefix(err.Error(), "yaml: ")}}
|
||||
}
|
||||
out := make([]Problem, 0, len(typed.Errors))
|
||||
for _, e := range typed.Errors {
|
||||
if m := unknownField.FindStringSubmatch(e); m != nil {
|
||||
out = append(out, Problem{Path: m[1], Message: "unknown field " + m[2]})
|
||||
continue
|
||||
}
|
||||
out = append(out, Problem{Path: "document", Message: e})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// checker collects the problems found in a document's steps.
|
||||
type checker struct {
|
||||
problems []Problem
|
||||
ids map[string]bool
|
||||
proposes, approves, ran bool
|
||||
}
|
||||
|
||||
func (c *checker) add(path, format string, args ...any) {
|
||||
c.problems = append(c.problems, Problem{Path: path, Message: fmt.Sprintf(format, args...)})
|
||||
}
|
||||
|
||||
func (d Document) validate() []Problem {
|
||||
c := &checker{ids: map[string]bool{}}
|
||||
if len(d.Steps) == 0 {
|
||||
c.add("steps", "a workflow needs steps")
|
||||
}
|
||||
for i, s := range d.Steps {
|
||||
c.step(fmt.Sprintf("steps[%d]", i), s)
|
||||
}
|
||||
if c.proposes && !c.approves {
|
||||
c.add("steps", "an agent or action step needs a wait_for_approval step, since judgement can hold what it proposes")
|
||||
}
|
||||
return c.problems
|
||||
}
|
||||
|
||||
func (c *checker) step(at string, s Step) {
|
||||
switch {
|
||||
case !stepID.MatchString(s.ID):
|
||||
c.add(at+".id", "a step id is lowercase letters, digits and hyphens, starting with a letter")
|
||||
case c.ids[s.ID]:
|
||||
c.add(at+".id", "step id %q is used twice", s.ID)
|
||||
}
|
||||
c.ids[s.ID] = true
|
||||
if n := s.kinds(); n != 1 {
|
||||
c.add(at, "a step has exactly one of agent, action, wait_for_approval, notify and sleep, found %d", n)
|
||||
return
|
||||
}
|
||||
c.kind(at, s)
|
||||
}
|
||||
|
||||
func (c *checker) kind(at string, s Step) {
|
||||
switch {
|
||||
case s.Agent != nil:
|
||||
c.proposes = true
|
||||
if c.ran {
|
||||
c.add(at+".agent", "only one agent step per workflow")
|
||||
}
|
||||
c.ran = true
|
||||
if _, ok := prompts.Agent(s.Agent.Prompt); !ok {
|
||||
c.add(at+".agent.prompt", "no prompt named %q", s.Agent.Prompt)
|
||||
}
|
||||
case s.Action != nil:
|
||||
c.proposes = true
|
||||
if s.Action.Connector == "" || s.Action.Name == "" {
|
||||
c.add(at+".action", "an action needs a connector and a name")
|
||||
}
|
||||
case s.WaitForApproval != nil:
|
||||
if c.approves {
|
||||
c.add(at+".wait_for_approval", "only one step can wait for approval")
|
||||
}
|
||||
c.approves = true
|
||||
case s.Sleep != nil:
|
||||
if dur, err := time.ParseDuration(s.Sleep.For); err != nil || dur <= 0 || dur > maxSleep {
|
||||
c.add(at+".sleep.for", "a duration between 1s and 24h, such as 30s")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s Step) kinds() int {
|
||||
n := 0
|
||||
for _, set := range []bool{s.Agent != nil, s.Action != nil, s.WaitForApproval != nil, s.Notify != nil, s.Sleep != nil} {
|
||||
if set {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// hasAgent reports whether any step runs an agent, and whether one uses memory.
|
||||
func (d Document) hasAgent() (agent, memory bool) {
|
||||
for _, s := range d.Steps {
|
||||
if s.Agent != nil {
|
||||
agent = true
|
||||
memory = memory || s.Agent.Memory
|
||||
}
|
||||
}
|
||||
return agent, memory
|
||||
}
|
||||
|
||||
// Describe says in plain language what each step does.
|
||||
func (d Document) Describe() []string {
|
||||
out := make([]string, 0, len(d.Steps))
|
||||
for _, s := range d.Steps {
|
||||
switch {
|
||||
case s.Agent != nil:
|
||||
line := "An agent investigates the event, looking things up and proposing actions, each judged before it runs"
|
||||
if s.Agent.Memory {
|
||||
line += "; it recalls earlier incidents and files what it found"
|
||||
}
|
||||
out = append(out, line)
|
||||
case s.Action != nil:
|
||||
out = append(out, fmt.Sprintf("Runs %s %s once judged", s.Action.Connector, s.Action.Name))
|
||||
case s.WaitForApproval != nil:
|
||||
out = append(out, "Asks a person before anything judgement holds runs")
|
||||
case s.Notify != nil:
|
||||
out = append(out, "Tells people what it found and did")
|
||||
case s.Sleep != nil:
|
||||
out = append(out, "Waits "+s.Sleep.For)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// duration is a sleep step's duration; validation has already accepted it.
|
||||
func (s SleepStep) duration() time.Duration {
|
||||
d, _ := time.ParseDuration(s.For)
|
||||
return d
|
||||
}
|
||||
|
||||
// normalise is the form two documents are compared in.
|
||||
func normalise(body string) string {
|
||||
return string(bytes.TrimSpace([]byte(strings.ReplaceAll(body, "\r\n", "\n"))))
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/deliveries"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
// dryOverlay is what a dry run carries instead of a stored workflow version:
|
||||
// the document it runs, and the policy it is judged under as if published.
|
||||
type dryOverlay struct {
|
||||
Document string `json:"document"`
|
||||
PolicyKind string `json:"policy_kind,omitzero"`
|
||||
Policy string `json:"policy,omitzero"`
|
||||
}
|
||||
|
||||
func (o *dryOverlay) policy() *judgement.PolicyOverlay {
|
||||
if o == nil || o.Policy == "" {
|
||||
return nil
|
||||
}
|
||||
return &judgement.PolicyOverlay{Kind: o.PolicyKind, Body: o.Policy}
|
||||
}
|
||||
|
||||
// readOverlay decodes the overlay a dry run carries.
|
||||
func readOverlay(raw jsontext.Value) (*dryOverlay, error) {
|
||||
var o dryOverlay
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
return nil, fmt.Errorf("read dry run overlay: %w", err)
|
||||
}
|
||||
return &o, nil
|
||||
}
|
||||
|
||||
// Dry reports whether the run is a dry run: every action is judged and none
|
||||
// that changes something is executed, nobody is asked and nothing is sent.
|
||||
func (rc *Context) Dry() bool { return rc.run.Dry }
|
||||
|
||||
// mayExecute reports whether judgement's decision may be carried out in this
|
||||
// run. A dry run carries out only what reads.
|
||||
func (rc *Context) mayExecute(d judgement.Decision) bool {
|
||||
return !rc.run.Dry || d.Action.Effect.ReadOnly
|
||||
}
|
||||
|
||||
// request asks judgement to decide an action of this run.
|
||||
func (rc *Context) request(key, connector, action string, input jsontext.Value) judgement.Request {
|
||||
diagnosis, evidence := rc.Rationale()
|
||||
return judgement.Request{
|
||||
RunID: rc.run.ID, Key: key, Connector: connector, Action: action, Input: input,
|
||||
Workflow: rc.run.Workflow, Dry: rc.run.Dry, Overlay: rc.overlay.policy(),
|
||||
Diagnosis: diagnosis, Evidence: evidence,
|
||||
}
|
||||
}
|
||||
|
||||
// DryRequest starts a dry run of a document that is not yet published.
|
||||
type DryRequest struct {
|
||||
OrgID string
|
||||
// Workflow is the name the document would be published under; hard rules
|
||||
// see it as the principal.
|
||||
Workflow string
|
||||
Document string
|
||||
PolicyKind string
|
||||
Policy string
|
||||
Input jsontext.Value
|
||||
}
|
||||
|
||||
// StartDry queues a dry run of a document and returns its run's id. A document
|
||||
// that doesn't validate starts nothing.
|
||||
func (e *Engine) StartDry(ctx context.Context, req DryRequest) (uuid.UUID, error) {
|
||||
if _, err := ParseDocument(req.Document); err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
overlay, err := json.Marshal(dryOverlay{Document: req.Document, PolicyKind: req.PolicyKind, Policy: req.Policy})
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("encode overlay: %w", err)
|
||||
}
|
||||
input := req.Input
|
||||
if len(input) == 0 {
|
||||
input = jsontext.Value("null")
|
||||
}
|
||||
return e.db.Start(ctx, store.NewRun{
|
||||
OrgID: req.OrgID, Workflow: req.Workflow, Input: input, Dry: true, Overlay: overlay,
|
||||
})
|
||||
}
|
||||
|
||||
// The outcomes a dry run reports for an action.
|
||||
const (
|
||||
// DryRan is an action that only reads, which ran for real.
|
||||
DryRan = "ran"
|
||||
// DryWouldRun is an action that changes something and judgement lets run
|
||||
// unattended. It was not executed.
|
||||
DryWouldRun = "would_run"
|
||||
// DryWouldAsk is an action judgement holds for a person.
|
||||
DryWouldAsk = "would_ask"
|
||||
// DryRefused is an action judgement or a hard rule refuses.
|
||||
DryRefused = "refused"
|
||||
)
|
||||
|
||||
// DryAction is one action a dry run judged.
|
||||
type DryAction struct {
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
Summary string `json:"summary,omitzero"`
|
||||
Level string `json:"level"`
|
||||
Outcome string `json:"outcome"`
|
||||
// Why is what held or refused the action.
|
||||
Why string `json:"why,omitzero"`
|
||||
// Ask is the people who would be asked, in order.
|
||||
Ask []string `json:"ask,omitzero"`
|
||||
}
|
||||
|
||||
// DryNotice is the message a dry run would have sent.
|
||||
type DryNotice struct {
|
||||
Title string `json:"title"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// DryReport is what a dry run did: every action it judged and what came of it.
|
||||
type DryReport struct {
|
||||
RunID uuid.UUID `json:"run_id"`
|
||||
State string `json:"state"`
|
||||
Error string `json:"error,omitzero"`
|
||||
Findings string `json:"findings,omitzero"`
|
||||
Actions []DryAction `json:"actions"`
|
||||
Notice *DryNotice `json:"notice,omitzero"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// DryReport reads what dry run id did so far. ok is false when there is no
|
||||
// such dry run.
|
||||
func (e *Engine) DryReport(ctx context.Context, orgID string, id uuid.UUID) (rep DryReport, ok bool, err error) {
|
||||
run, found, err := e.db.Detail(ctx, orgID, id)
|
||||
if err != nil || !found || !run.Dry {
|
||||
return DryReport{}, false, err
|
||||
}
|
||||
steps, err := e.db.RecordedSteps(ctx, id)
|
||||
if err != nil {
|
||||
return DryReport{}, false, err
|
||||
}
|
||||
decisions, err := e.deps.Judge.ForRun(ctx, id)
|
||||
if err != nil {
|
||||
return DryReport{}, false, err
|
||||
}
|
||||
rep = DryReport{RunID: id, State: run.State, Error: run.Error.String, At: run.CreatedAt, Actions: make([]DryAction, 0, len(decisions))}
|
||||
var result AgentResult
|
||||
if json.Unmarshal(run.Result, &result) == nil {
|
||||
rep.Findings = result.Answer
|
||||
}
|
||||
asked := map[string][]string{}
|
||||
for _, s := range steps {
|
||||
if key, isAsk := strings.CutPrefix(s.Name, "would-ask:"); isAsk {
|
||||
var path []governance.PathStep
|
||||
if json.Unmarshal(s.Output, &path) == nil {
|
||||
asked[key] = pathNames(path)
|
||||
}
|
||||
}
|
||||
if s.Name == "notify" && s.Outcome == store.OutcomeDone {
|
||||
var n deliveries.Notification
|
||||
if json.Unmarshal(s.Output, &n) == nil {
|
||||
rep.Notice = &DryNotice{Title: n.Title, Body: n.Body}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range decisions {
|
||||
rep.Actions = append(rep.Actions, dryAction(d, asked[d.Key]))
|
||||
}
|
||||
return rep, true, nil
|
||||
}
|
||||
|
||||
func dryAction(d judgement.Decision, ask []string) DryAction {
|
||||
a := DryAction{
|
||||
Action: d.Action.Connector + " " + d.Action.Name,
|
||||
Target: d.Action.Target.Kind + ":" + d.Action.Target.ID,
|
||||
Summary: d.Action.Effect.Summary,
|
||||
Level: d.Level.String(),
|
||||
Ask: ask,
|
||||
}
|
||||
switch {
|
||||
case d.Level == judgement.Never:
|
||||
a.Outcome, a.Why = DryRefused, firstText(d.Reason, "refused")
|
||||
case d.Level.Runs() && d.Action.Effect.ReadOnly:
|
||||
a.Outcome = DryRan
|
||||
case d.Level.Runs():
|
||||
a.Outcome = DryWouldRun
|
||||
default:
|
||||
a.Outcome, a.Why = DryWouldAsk, findingsOf(d)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func pathNames(path []governance.PathStep) []string {
|
||||
names := make([]string, 0, len(path))
|
||||
for _, s := range path {
|
||||
names = append(names, firstText(s.Name, s.Person))
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// settleDry records who each held action would be put to, and asks nobody.
|
||||
func settleDry(rc *Context, judge Judge, settings Settings) ([]approvalOutcome, error) {
|
||||
held, err := rc.Step("proposals", func(ctx context.Context, _ string) ([]judgement.Decision, error) {
|
||||
decisions, err := judge.ForRun(ctx, rc.RunID())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return heldByJudgement(decisions), nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]approvalOutcome, 0, len(held))
|
||||
for _, d := range held {
|
||||
path, err := rc.Step("would-ask:"+d.Key, func(ctx context.Context, _ string) ([]governance.PathStep, error) {
|
||||
return settings.EscalationPath(ctx, d.Owner(), d.Level == judgement.Escalate)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, approvalOutcome{
|
||||
Action: d.Action.Connector + " " + d.Action.Name,
|
||||
Target: d.Action.Target.Kind + ":" + d.Action.Target.ID,
|
||||
State: wouldAsk(path), digest: d.Action.Digest,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// wouldAsk says who a request would go to.
|
||||
func wouldAsk(path []governance.PathStep) string {
|
||||
if len(path) == 0 {
|
||||
return "would ask the channels"
|
||||
}
|
||||
return "would ask " + strings.Join(pathNames(path), ", then ")
|
||||
}
|
||||
|
||||
var errNoDeps = errors.New("the engine has no steps to run: Use was not called")
|
||||
@@ -0,0 +1,76 @@
|
||||
package runs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
type decidingJudge struct {
|
||||
decision judgement.Decision
|
||||
executed int
|
||||
}
|
||||
|
||||
func (j *decidingJudge) Assess(context.Context, judgement.Request) (judgement.Decision, error) {
|
||||
return j.decision, nil
|
||||
}
|
||||
|
||||
func (j *decidingJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
|
||||
j.executed++
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func (j *decidingJudge) ExecuteApproved(context.Context, judgement.Decision, judgement.Approval, jsontext.Value, string) (string, error) {
|
||||
j.executed++
|
||||
return "done", nil
|
||||
}
|
||||
|
||||
func (j *decidingJudge) ForRun(context.Context, uuid.UUID) ([]judgement.Decision, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type anyBox struct{}
|
||||
|
||||
func (anyBox) Declarations() []connectors.Declaration { return nil }
|
||||
func (anyBox) Tool(string) (connectors.Tool, bool) { return nil, true }
|
||||
|
||||
// TestDryRunNeverExecutesAChangingAction guards the dry run's promise: an
|
||||
// action that changes something is judged and never executed, however freely
|
||||
// judgement would let it run, while one that only reads runs for real.
|
||||
func TestDryRunNeverExecutesAChangingAction(t *testing.T) {
|
||||
decided := func(level judgement.Level, readOnly bool) judgement.Decision {
|
||||
d := judgement.Decision{Level: level}
|
||||
d.Action.Effect.ReadOnly = readOnly
|
||||
return d
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
dry bool
|
||||
decision judgement.Decision
|
||||
wantExecuted int
|
||||
}{
|
||||
{name: "dry: a changing action at act quietly is not executed", dry: true, decision: decided(judgement.ActQuietly, false)},
|
||||
{name: "dry: a changing action at act and tell is not executed", dry: true, decision: decided(judgement.ActAndTell, false)},
|
||||
{name: "dry: a read-only action runs for real", dry: true, decision: decided(judgement.ActQuietly, true), wantExecuted: 1},
|
||||
{name: "live: a changing action at act quietly runs", decision: decided(judgement.ActQuietly, false), wantExecuted: 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
judge := &decidingJudge{decision: tt.decision}
|
||||
rc := newContext(t.Context(), &fakeWork{}, store.Run{ID: uuid.New(), Dry: tt.dry}, uuid.New(), nil)
|
||||
decl := connectors.Declaration{Connector: "c", Kind: connectors.KindAction, Name: "restart"}
|
||||
if _, err := call(t.Context(), anyBox{}, judge, rc, decl, jsontext.Value(`{}`)); err != nil {
|
||||
t.Fatalf("call: %v", err)
|
||||
}
|
||||
if judge.executed != tt.wantExecuted {
|
||||
t.Fatalf("executed %d times, want %d", judge.executed, tt.wantExecuted)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,10 @@ type Run struct {
|
||||
Version int `db:"workflow_version"`
|
||||
Input jsontext.Value `db:"input"`
|
||||
Attempts int `db:"attempts"`
|
||||
// Dry marks a dry run, and Overlay is the document and policy it runs
|
||||
// under; a dry run pins no stored version.
|
||||
Dry bool `db:"dry"`
|
||||
Overlay jsontext.Value `db:"overlay"`
|
||||
}
|
||||
|
||||
// Step is one recorded step of a run.
|
||||
@@ -70,18 +74,29 @@ func New(db *sqlx.DB) *Store {
|
||||
return &Store{db: db}
|
||||
}
|
||||
|
||||
// NewRun is a run to queue.
|
||||
type NewRun struct {
|
||||
OrgID string
|
||||
Workflow string
|
||||
Version int
|
||||
StartKey string
|
||||
Input jsontext.Value
|
||||
Dry bool
|
||||
Overlay jsontext.Value
|
||||
}
|
||||
|
||||
// Start inserts a queued run. A run with the same organisation and start key
|
||||
// is not inserted again; its id is returned instead.
|
||||
func (s *Store) Start(ctx context.Context, orgID, workflow string, version int, startKey string, input jsontext.Value) (uuid.UUID, error) {
|
||||
key := sql.NullString{String: startKey, Valid: startKey != ""}
|
||||
func (s *Store) Start(ctx context.Context, n NewRun) (uuid.UUID, error) {
|
||||
key := sql.NullString{String: n.StartKey, Valid: n.StartKey != ""}
|
||||
var id uuid.UUID
|
||||
err := s.db.GetContext(ctx, &id, `
|
||||
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'queued')
|
||||
INSERT INTO runs.runs (id, org_id, workflow, workflow_version, start_key, input, state, dry, overlay)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'queued', $7, $8)
|
||||
ON CONFLICT (org_id, start_key) DO NOTHING
|
||||
RETURNING id`, uuid.New(), orgID, workflow, version, key, input)
|
||||
RETURNING id`, uuid.New(), n.OrgID, n.Workflow, n.Version, key, n.Input, n.Dry, nullJSON(n.Overlay))
|
||||
if errors.Is(err, sql.ErrNoRows) && key.Valid {
|
||||
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, orgID, startKey)
|
||||
err = s.db.GetContext(ctx, &id, `SELECT id FROM runs.runs WHERE org_id = $1 AND start_key = $2`, n.OrgID, n.StartKey)
|
||||
}
|
||||
if err != nil {
|
||||
return uuid.Nil, fmt.Errorf("start run: %w", err)
|
||||
@@ -107,7 +122,7 @@ func (s *Store) Claim(ctx context.Context, owner string, token uuid.UUID, ttl ti
|
||||
ORDER BY created_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1)
|
||||
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts`, token, owner, ttl.Seconds())
|
||||
RETURNING r.id, r.org_id, r.workflow, r.workflow_version, r.input, r.attempts, r.dry, r.overlay`, token, owner, ttl.Seconds())
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Run{}, false, nil
|
||||
}
|
||||
@@ -261,6 +276,7 @@ type Summary struct {
|
||||
Error sql.NullString `db:"error"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
FinishedAt sql.NullTime `db:"finished_at"`
|
||||
Dry bool `db:"dry"`
|
||||
}
|
||||
|
||||
// Detail is a run with its input and result. A run that failed has no result,
|
||||
@@ -282,8 +298,8 @@ type Recorded struct {
|
||||
func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary, error) {
|
||||
var out []Summary
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at FROM runs.runs
|
||||
WHERE org_id = $1 ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry FROM runs.runs
|
||||
WHERE org_id = $1 AND NOT dry ORDER BY created_at DESC LIMIT $2`, orgID, limit); err != nil {
|
||||
return nil, fmt.Errorf("list runs: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
@@ -294,7 +310,7 @@ func (s *Store) Recent(ctx context.Context, orgID string, limit int) ([]Summary,
|
||||
func (s *Store) Detail(ctx context.Context, orgID string, id uuid.UUID) (Detail, bool, error) {
|
||||
var d Detail
|
||||
err := s.db.GetContext(ctx, &d, `
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, input,
|
||||
SELECT id, workflow, workflow_version, state, attempts, error, created_at, finished_at, dry, input,
|
||||
coalesce(result, 'null') AS result
|
||||
FROM runs.runs WHERE org_id = $1 AND id = $2`, orgID, id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrNoWorkflow means no workflow is stored under that name and version.
|
||||
var ErrNoWorkflow = errors.New("no such workflow")
|
||||
|
||||
// Workflow is one stored version of a workflow document.
|
||||
type Workflow struct {
|
||||
Name string `db:"name"`
|
||||
Version int `db:"version"`
|
||||
Body string `db:"body"`
|
||||
CreatedBy string `db:"created_by"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
}
|
||||
|
||||
// Workflow returns a version of a workflow, or its latest when version is zero.
|
||||
func (s *Store) Workflow(ctx context.Context, orgID, name string, version int) (Workflow, error) {
|
||||
var w Workflow
|
||||
err := s.db.GetContext(ctx, &w, `
|
||||
SELECT name, version, body, created_by, created_at FROM runs.workflows
|
||||
WHERE org_id = $1 AND name = $2 AND ($3 = 0 OR version = $3)
|
||||
ORDER BY version DESC LIMIT 1`, orgID, name, version)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Workflow{}, ErrNoWorkflow
|
||||
}
|
||||
if err != nil {
|
||||
return Workflow{}, fmt.Errorf("read workflow: %w", err)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Workflows returns the latest version of every workflow.
|
||||
func (s *Store) Workflows(ctx context.Context, orgID string) ([]Workflow, error) {
|
||||
var out []Workflow
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT DISTINCT ON (name) name, version, body, created_by, created_at FROM runs.workflows
|
||||
WHERE org_id = $1 ORDER BY name, version DESC`, orgID); err != nil {
|
||||
return nil, fmt.Errorf("list workflows: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AddWorkflow stores body as the next version of a workflow and returns it.
|
||||
func (s *Store) AddWorkflow(ctx context.Context, orgID, name, body, by string) (Workflow, error) {
|
||||
var w Workflow
|
||||
err := s.db.GetContext(ctx, &w, `
|
||||
INSERT INTO runs.workflows (org_id, name, version, body, created_by)
|
||||
SELECT $1, $2, COALESCE(max(version), 0) + 1, $3, $4 FROM runs.workflows WHERE org_id = $1 AND name = $2
|
||||
RETURNING name, version, body, created_by, created_at`, orgID, name, body, by)
|
||||
if err != nil {
|
||||
return Workflow{}, fmt.Errorf("store workflow: %w", err)
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
+317
-94
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/samcharles93/ai-sdk/agentloop"
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
"github.com/samcharles93/ai-sdk/core"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/prompts"
|
||||
@@ -84,101 +85,318 @@ func stepBudget(ctx context.Context, settings Settings) (agentloop.Budget, error
|
||||
return budget, nil
|
||||
}
|
||||
|
||||
// Investigate is the workflow that has an agent look into the event that
|
||||
// started the run and report its findings. The agent model and the tools are
|
||||
// pinned in the run's first steps, so a resumed run asks the same model with
|
||||
// the same tools even if settings or connectors changed meanwhile. An action
|
||||
// the agent proposes is judged as it calls it; one judgement holds is put to a
|
||||
// person, and the run waits for the answer before running it.
|
||||
func Investigate(settings Settings, box Toolbox, judge Judge, ask Approver, notify Notifier, mem Memory) Workflow {
|
||||
// Deps is what a workflow document's steps use: the organisation's settings,
|
||||
// the connectors and tools an agent may call, judgement, approvals, delivery
|
||||
// and memory.
|
||||
type Deps struct {
|
||||
Settings Settings
|
||||
Toolbox Toolbox
|
||||
Judge Judge
|
||||
Approver Approver
|
||||
Notifier Notifier
|
||||
Memory Memory
|
||||
}
|
||||
|
||||
// Workflow is a document ready to run: the code that executes its steps for
|
||||
// one run. It chooses its next step only from the document, the run's input
|
||||
// and the results of earlier steps.
|
||||
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
|
||||
|
||||
// pinned is what a run's first steps fix, so a resumed run asks the same model
|
||||
// with the same tools even if settings or connectors changed meanwhile.
|
||||
type pinned struct {
|
||||
model governance.Model
|
||||
threshold int
|
||||
decls []connectors.Declaration
|
||||
provider chat.Provider
|
||||
info chat.ModelInfo
|
||||
}
|
||||
|
||||
// progress is what the steps of one execution have produced so far.
|
||||
type progress struct {
|
||||
pin pinned
|
||||
agent *AgentResult
|
||||
memory bool
|
||||
remembered bool
|
||||
outcomes []approvalOutcome
|
||||
review memory.Review
|
||||
// stopped is an action whose outcome can't be confirmed: nothing further
|
||||
// is proposed or run, and the owner is told.
|
||||
stopped bool
|
||||
}
|
||||
|
||||
// Compile returns the code that runs doc. The same steps run live and dry; a
|
||||
// dry run (rc.Dry) judges every action and executes only those that read.
|
||||
func (d Deps) Compile(doc Document) Workflow {
|
||||
return func(rc *Context, input jsontext.Value) (jsontext.Value, error) {
|
||||
model, err := rc.Step("agent-model", func(ctx context.Context, _ string) (governance.Model, error) {
|
||||
return settings.AgentModel(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
p := &progress{outcomes: []approvalOutcome{}}
|
||||
if agent, mem := doc.hasAgent(); agent {
|
||||
if err := d.pin(rc, p, mem); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
threshold, err := rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
||||
return settings.RecurrenceThreshold(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
for _, s := range doc.Steps {
|
||||
if p.stopped && s.Notify == nil {
|
||||
continue
|
||||
}
|
||||
if err := d.step(rc, p, s, input); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
decls, err := rc.Step("agent-tools", func(context.Context, string) ([]connectors.Declaration, error) {
|
||||
return box.Declarations(), nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
provider, info, err := model.Provider(rc.Context())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
budget, err := stepBudget(rc.Context(), settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := rc.Agent(AgentRequest{
|
||||
Name: "investigate",
|
||||
Provider: provider,
|
||||
Model: model.Model,
|
||||
ModelInfo: info,
|
||||
System: prompts.Investigate(),
|
||||
Prompt: string(input),
|
||||
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||
Budget: budget,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res, err = followThrough(rc, judge, decls, input, res, AgentRequest{
|
||||
Provider: provider, Model: model.Model, ModelInfo: info,
|
||||
Tools: withMemory(toolSet(box, judge, rc, decls), mem, rc.RunID()),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||
return stopUnknown(rc, notify, res)
|
||||
}
|
||||
if err := recordCause(rc, mem, input, res, AgentRequest{Provider: provider, Model: model.Model, ModelInfo: info}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outcomes, err := settleProposals(rc, judge, ask, settings)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
review, err := remember(rc, mem, judge, outcomes, threshold)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conclude(rc, notify, res, outcomes, review)
|
||||
return p.result()
|
||||
}
|
||||
}
|
||||
|
||||
// stopUnknown ends an investigation that stopped on an action whose outcome
|
||||
// can't be confirmed. Nothing further is proposed or run; the owner is told.
|
||||
func stopUnknown(rc *Context, notify Notifier, res AgentResult) (jsontext.Value, error) {
|
||||
return conclude(rc, notify, res, nil, memory.Review{})
|
||||
func (d Deps) step(rc *Context, p *progress, s Step, input jsontext.Value) error {
|
||||
switch {
|
||||
case s.Agent != nil:
|
||||
return d.agent(rc, p, s, input)
|
||||
case s.Action != nil:
|
||||
return d.action(rc, p, s, input)
|
||||
case s.WaitForApproval != nil:
|
||||
return d.approve(rc, p)
|
||||
case s.Notify != nil:
|
||||
return d.notify(rc, p, s)
|
||||
default:
|
||||
return d.sleep(rc, s)
|
||||
}
|
||||
}
|
||||
|
||||
// conclude tells the owner what the investigation found and returns it as the
|
||||
// run's output, failing the run when the agent did not complete.
|
||||
func conclude(rc *Context, notify Notifier, res AgentResult, outcomes []approvalOutcome, review memory.Review) (jsontext.Value, error) {
|
||||
if _, err := rc.Step("notify", func(ctx context.Context, key string) (bool, error) {
|
||||
return true, notify.Notify(ctx, findings(rc.RunID(), key, res, outcomes, review))
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := json.Marshal(res)
|
||||
func (d Deps) pin(rc *Context, p *progress, withMemory bool) error {
|
||||
model, err := rc.Step("agent-model", func(ctx context.Context, _ string) (governance.Model, error) {
|
||||
return d.Settings.AgentModel(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode findings: %w", err)
|
||||
return err
|
||||
}
|
||||
if !res.Completed {
|
||||
return out, fmt.Errorf("investigation stopped: %s", res.StopReason)
|
||||
if withMemory {
|
||||
if p.pin.threshold, err = rc.Step("recurrence-threshold", func(ctx context.Context, _ string) (int, error) {
|
||||
return d.Settings.RecurrenceThreshold(ctx)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
decls, err := rc.Step("agent-tools", func(context.Context, string) ([]connectors.Declaration, error) {
|
||||
return d.Toolbox.Declarations(), nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
provider, info, err := model.Provider(rc.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.pin.model, p.pin.decls, p.pin.provider, p.pin.info = model, decls, provider, info
|
||||
return nil
|
||||
}
|
||||
|
||||
// agent has the agent look into the event. An action it proposes is judged as
|
||||
// it calls it; the ones judgement holds wait for the approval step.
|
||||
func (d Deps) agent(rc *Context, p *progress, s Step, input jsontext.Value) error {
|
||||
system, _ := prompts.Agent(s.Agent.Prompt)
|
||||
budget, err := stepBudget(rc.Context(), d.Settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tools := func() core.ToolSet {
|
||||
set := toolSet(d.Toolbox, d.Judge, rc, p.pin.decls)
|
||||
if s.Agent.Memory {
|
||||
set = withMemory(set, d.Memory, rc.RunID())
|
||||
}
|
||||
return set
|
||||
}
|
||||
base := AgentRequest{Provider: p.pin.provider, Model: p.pin.model.Model, ModelInfo: p.pin.info, System: system, Tools: tools()}
|
||||
first := base
|
||||
first.Name, first.Prompt, first.Budget = s.ID, string(input), budget
|
||||
res, err := rc.Agent(first)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
followUp := base
|
||||
followUp.Tools = tools()
|
||||
if res, err = followThrough(rc, d.Judge, p.pin.decls, input, res, followUp); err != nil {
|
||||
return err
|
||||
}
|
||||
p.agent = &res
|
||||
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||
p.stopped = true
|
||||
return nil
|
||||
}
|
||||
p.memory = s.Agent.Memory
|
||||
if p.memory && !rc.Dry() {
|
||||
return recordCause(rc, d.Memory, input, res, base)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// action judges one action the document names and runs it if judgement lets it
|
||||
// run unattended. One judgement holds goes to the approval step; one it refuses
|
||||
// is reported.
|
||||
func (d Deps) action(rc *Context, p *progress, s Step, input jsontext.Value) error {
|
||||
in, err := resolveInput(s.Action.Input, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("step %q: %w", s.ID, err)
|
||||
}
|
||||
decision, err := rc.Step("judge:"+s.ID, func(ctx context.Context, key string) (judgement.Decision, error) {
|
||||
return d.Judge.Assess(ctx, rc.request(key, s.Action.Connector, s.Action.Name, in))
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !decision.Level.Runs() || !rc.mayExecute(decision) {
|
||||
return nil
|
||||
}
|
||||
out := approvalOutcome{
|
||||
Action: decision.Action.Connector + " " + decision.Action.Name,
|
||||
Target: decision.Action.Target.Kind + ":" + decision.Action.Target.ID,
|
||||
State: "ran", digest: decision.Action.Digest,
|
||||
}
|
||||
out.Result, out.Error, out.Unknown = runAction(rc, d.Judge, decision, nil)
|
||||
p.outcomes = append(p.outcomes, out)
|
||||
p.stopped = p.stopped || out.Unknown
|
||||
return nil
|
||||
}
|
||||
|
||||
// approve puts what judgement held to people and runs what they allow. In a dry
|
||||
// run nobody is asked: it records who would be.
|
||||
func (d Deps) approve(rc *Context, p *progress) error {
|
||||
var (
|
||||
out []approvalOutcome
|
||||
err error
|
||||
)
|
||||
if rc.Dry() {
|
||||
out, err = settleDry(rc, d.Judge, d.Settings)
|
||||
} else {
|
||||
out, err = settleProposals(rc, d.Judge, d.Approver, d.Settings)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.outcomes = append(p.outcomes, out...)
|
||||
return nil
|
||||
}
|
||||
|
||||
// notify tells people what the run found, after filing it in memory. In a dry
|
||||
// run it records what would be sent and sends nothing.
|
||||
func (d Deps) notify(rc *Context, p *progress, s Step) error {
|
||||
if p.memory && !p.remembered && !p.stopped && !rc.Dry() {
|
||||
review, err := remember(rc, d.Memory, d.Judge, p.outcomes, p.pin.threshold)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.review, p.remembered = review, true
|
||||
}
|
||||
if rc.Dry() {
|
||||
_, err := rc.Step("notify", func(_ context.Context, key string) (deliveries.Notification, error) {
|
||||
return findings(rc.RunID(), key, p.agent, s.Notify.Title, p.outcomes, p.review), nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
_, err := rc.Step("notify", func(ctx context.Context, key string) (bool, error) {
|
||||
return true, d.Notifier.Notify(ctx, findings(rc.RunID(), key, p.agent, s.Notify.Title, p.outcomes, p.review))
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// sleep suspends the run. A dry run records the wait and doesn't take it.
|
||||
func (d Deps) sleep(rc *Context, s Step) error {
|
||||
if rc.Dry() {
|
||||
_, err := rc.Step("sleep-skipped", func(context.Context, string) (string, error) {
|
||||
return s.Sleep.For, nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
return rc.Sleep(s.Sleep.duration())
|
||||
}
|
||||
|
||||
// result is the run's output: the agent's findings, or what became of the
|
||||
// actions when no agent ran. An agent that did not complete fails the run once
|
||||
// the owner has been told.
|
||||
func (p *progress) result() (jsontext.Value, error) {
|
||||
var v any = p.outcomes
|
||||
if p.agent != nil {
|
||||
v = *p.agent
|
||||
}
|
||||
out, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode result: %w", err)
|
||||
}
|
||||
if p.agent != nil && !p.agent.Completed {
|
||||
return out, fmt.Errorf("investigation stopped: %s", p.agent.StopReason)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolveInput builds an action's input from the document, replacing a string
|
||||
// written {{event.path}} with the value at that path of the run's input.
|
||||
func resolveInput(in map[string]any, input jsontext.Value) (jsontext.Value, error) {
|
||||
var root any
|
||||
if len(input) > 0 {
|
||||
if err := json.Unmarshal(input, &root); err != nil {
|
||||
return nil, fmt.Errorf("read run input: %w", err)
|
||||
}
|
||||
}
|
||||
var walk func(v any) (any, error)
|
||||
walk = func(v any) (any, error) {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
out := make(map[string]any, len(x))
|
||||
for k, e := range x {
|
||||
r, err := walk(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = r
|
||||
}
|
||||
return out, nil
|
||||
case []any:
|
||||
out := make([]any, len(x))
|
||||
for i, e := range x {
|
||||
r, err := walk(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[i] = r
|
||||
}
|
||||
return out, nil
|
||||
case string:
|
||||
path, ok := strings.CutSuffix(strings.TrimPrefix(strings.TrimSpace(x), "{{"), "}}")
|
||||
if !ok || !strings.HasPrefix(strings.TrimSpace(x), "{{") {
|
||||
return x, nil
|
||||
}
|
||||
return lookup(root, strings.TrimSpace(path))
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
resolved, err := walk(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if in == nil {
|
||||
resolved = map[string]any{}
|
||||
}
|
||||
out, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode action input: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// lookup returns the value at a dotted path.
|
||||
func lookup(root any, path string) (any, error) {
|
||||
cur := root
|
||||
for part := range strings.SplitSeq(path, ".") {
|
||||
m, ok := cur.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("{{%s}}: nothing at %q", path, part)
|
||||
}
|
||||
if cur, ok = m[part]; !ok {
|
||||
return nil, fmt.Errorf("{{%s}}: nothing at %q", path, part)
|
||||
}
|
||||
}
|
||||
return cur, nil
|
||||
}
|
||||
|
||||
// begun records that an action is about to run. It fails with an unknown
|
||||
// outcome when an earlier attempt already started it, so the action is not run
|
||||
// again.
|
||||
@@ -223,7 +441,7 @@ func followThrough(rc *Context, judge Judge, decls []connectors.Declaration, inp
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
req.Name, req.System, req.Prompt, req.Budget = "follow-through", prompts.Investigate()+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
||||
req.Name, req.System, req.Prompt, req.Budget = "follow-through", req.System+"\n\n"+prompts.FollowThrough(), string(prompt), followThroughBudget
|
||||
again, err := rc.Agent(req)
|
||||
if err == nil && again.StopReason == agentloop.StopOutcomeUnknown {
|
||||
return again, nil
|
||||
@@ -317,11 +535,7 @@ func settleOne(rc *Context, judge Judge, ask Approver, settings Settings, d judg
|
||||
}
|
||||
out.By = answer.By
|
||||
fresh, err := rc.Step("judge-again:"+d.Key, func(ctx context.Context, key string) (judgement.Decision, error) {
|
||||
diagnosis, evidence := rc.Rationale()
|
||||
return judge.Assess(ctx, judgement.Request{
|
||||
RunID: rc.RunID(), Key: key, Connector: d.Action.Connector, Action: d.Action.Name, Input: d.Action.Input,
|
||||
Workflow: rc.Workflow(), Diagnosis: diagnosis, Evidence: evidence,
|
||||
})
|
||||
return judge.Assess(ctx, rc.request(key, d.Action.Connector, d.Action.Name, d.Action.Input))
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
@@ -536,9 +750,19 @@ func heldByJudgement(decisions []judgement.Decision) []judgement.Decision {
|
||||
// waitKey is the run key an approval answer wakes.
|
||||
func waitKey(key string) string { return "approval:" + key }
|
||||
|
||||
// findings is the notification an investigation ends with, finished or not.
|
||||
func findings(runID uuid.UUID, key string, res AgentResult, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
||||
// findings is the notification a run ends with, finished or not.
|
||||
func findings(runID uuid.UUID, key string, agent *AgentResult, title string, outcomes []approvalOutcome, review memory.Review) deliveries.Notification {
|
||||
if agent == nil {
|
||||
if title == "" {
|
||||
title = "Run finished"
|
||||
}
|
||||
return deliveries.Notification{RunID: runID, Key: key, Title: title, Body: outcomeLines(outcomes)}
|
||||
}
|
||||
res := *agent
|
||||
n := deliveries.Notification{RunID: runID, Key: key, Title: "Investigation finished", Body: res.Answer}
|
||||
if title != "" {
|
||||
n.Title = title
|
||||
}
|
||||
if res.StopReason == agentloop.StopOutcomeUnknown {
|
||||
n.Title = "An action's outcome is unknown"
|
||||
n.Body = "An action was started and its outcome can't be confirmed. It may have taken effect and is not run again; check its target. " + res.Detail
|
||||
@@ -633,14 +857,13 @@ func call(ctx context.Context, box Toolbox, judge Judge, rc *Context, d connecto
|
||||
// is proposed to a person when the investigation ends, so it comes back to
|
||||
// the agent as something waiting rather than a failure.
|
||||
key := core.IdempotencyKey(ctx)
|
||||
diagnosis, evidence := rc.Rationale()
|
||||
decision, err := judge.Assess(ctx, judgement.Request{
|
||||
RunID: rc.RunID(), Key: key, Connector: d.Connector, Action: d.Name, Input: input, Workflow: rc.Workflow(),
|
||||
Diagnosis: diagnosis, Evidence: evidence,
|
||||
})
|
||||
decision, err := judge.Assess(ctx, rc.request(key, d.Connector, d.Name, input))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if decision.Level.Runs() && !rc.mayExecute(decision) {
|
||||
return fmt.Sprintf("dry run: this would run now and was not executed: %s. Carry on as if it had succeeded.", decision.Action.Effect.Summary), nil
|
||||
}
|
||||
if decision.Level.Runs() {
|
||||
if err := begun(ctx, rc, key); err != nil {
|
||||
return "", err
|
||||
|
||||
+97
-43
@@ -1,5 +1,5 @@
|
||||
// Package runs is Archie's durable run engine. A run executes a registered
|
||||
// workflow step by step; each step's result is saved before the next starts,
|
||||
// Package runs is Archie's durable run engine. A run executes a stored
|
||||
// workflow document step by step; each step's result is saved before the next starts,
|
||||
// so after a crash the run replays from saved results without repeating a
|
||||
// finished step. Every write is fenced by the worker's lease.
|
||||
package runs
|
||||
@@ -19,17 +19,6 @@ import (
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
|
||||
// Workflow is the code a run executes. It must choose its next step only from
|
||||
// its input and the results of earlier steps; clock reads, randomness and
|
||||
// lookups belong inside steps.
|
||||
type Workflow func(rc *Context, input jsontext.Value) (jsontext.Value, error)
|
||||
|
||||
// registered is a workflow with the version its code declares.
|
||||
type registered struct {
|
||||
version int
|
||||
run Workflow
|
||||
}
|
||||
|
||||
// StartRequest names the workflow to run and the organisation it runs for.
|
||||
// A non-empty Key makes the start idempotent within the organisation.
|
||||
type StartRequest struct {
|
||||
@@ -61,12 +50,14 @@ type workerStore interface {
|
||||
|
||||
// Engine starts, executes and resumes runs.
|
||||
type Engine struct {
|
||||
db *store.Store
|
||||
work workerStore
|
||||
log *slog.Logger
|
||||
opts Options
|
||||
mu sync.RWMutex
|
||||
workflows map[string]registered
|
||||
db *store.Store
|
||||
work workerStore
|
||||
log *slog.Logger
|
||||
opts Options
|
||||
mu sync.RWMutex
|
||||
deps *Deps
|
||||
// load turns a claimed run into the code that executes it.
|
||||
load func(ctx context.Context, run store.Run) (Workflow, error)
|
||||
}
|
||||
|
||||
// New returns an engine on db.
|
||||
@@ -88,30 +79,76 @@ func newEngine(db *store.Store, work workerStore, log *slog.Logger, opts Options
|
||||
if opts.Owner == "" {
|
||||
opts.Owner = "archie"
|
||||
}
|
||||
return &Engine{db: db, work: work, log: log, opts: opts, workflows: map[string]registered{}}
|
||||
e := &Engine{db: db, work: work, log: log, opts: opts}
|
||||
e.load = e.loadDocument
|
||||
return e
|
||||
}
|
||||
|
||||
// Register makes a workflow available to runs by name. Raise version whenever
|
||||
// a change to wf would alter the steps an in-flight run replays: a run only
|
||||
// executes under the version it started with.
|
||||
func (e *Engine) Register(name string, version int, wf Workflow) {
|
||||
// Use gives the engine what a document's steps need. It must be called before
|
||||
// Work, and is separate from New because the modules those steps use are built
|
||||
// with the engine.
|
||||
func (e *Engine) Use(d Deps) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.workflows[name] = registered{version: version, run: wf}
|
||||
e.deps = &d
|
||||
}
|
||||
|
||||
// Start queues a run under the registered version of its workflow and returns
|
||||
// its id.
|
||||
func (e *Engine) steps() (Deps, error) {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
if e.deps == nil {
|
||||
return Deps{}, errNoDeps
|
||||
}
|
||||
return *e.deps, nil
|
||||
}
|
||||
|
||||
// Install stores body as the next version of the workflow name, unless it is
|
||||
// what the latest version already is, and returns the version in force. A body
|
||||
// that doesn't validate is stored as nothing.
|
||||
func (e *Engine) Install(ctx context.Context, orgID, name, body, by string) (int, error) {
|
||||
if _, err := ParseDocument(body); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
latest, err := e.db.Workflow(ctx, orgID, name, 0)
|
||||
switch {
|
||||
case err == nil && normalise(latest.Body) == normalise(body):
|
||||
return latest.Version, nil
|
||||
case err != nil && !errors.Is(err, store.ErrNoWorkflow):
|
||||
return 0, err
|
||||
}
|
||||
w, err := e.db.AddWorkflow(ctx, orgID, name, body, by)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return w.Version, nil
|
||||
}
|
||||
|
||||
// Has reports whether a workflow is stored under name.
|
||||
func (e *Engine) Has(ctx context.Context, orgID, name string) (bool, error) {
|
||||
_, err := e.db.Workflow(ctx, orgID, name, 0)
|
||||
if errors.Is(err, store.ErrNoWorkflow) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// Start queues a run under the latest stored version of its workflow and
|
||||
// returns its id.
|
||||
func (e *Engine) Start(ctx context.Context, req StartRequest) (uuid.UUID, error) {
|
||||
wf, ok := e.workflow(req.Workflow)
|
||||
if !ok {
|
||||
w, err := e.db.Workflow(ctx, req.OrgID, req.Workflow, 0)
|
||||
if errors.Is(err, store.ErrNoWorkflow) {
|
||||
return uuid.Nil, fmt.Errorf("no workflow named %q", req.Workflow)
|
||||
}
|
||||
if err != nil {
|
||||
return uuid.Nil, err
|
||||
}
|
||||
input := req.Input
|
||||
if len(input) == 0 {
|
||||
input = jsontext.Value("null")
|
||||
}
|
||||
return e.db.Start(ctx, req.OrgID, req.Workflow, wf.version, req.Key, input)
|
||||
return e.db.Start(ctx, store.NewRun{
|
||||
OrgID: req.OrgID, Workflow: req.Workflow, Version: w.Version, StartKey: req.Key, Input: input,
|
||||
})
|
||||
}
|
||||
|
||||
// Signal delivers a payload to a run waiting on key, now or later.
|
||||
@@ -197,26 +234,43 @@ func (e *Engine) invoke(ctx context.Context, run store.Run, token uuid.UUID) (re
|
||||
err = fmt.Errorf("panic: %v", p)
|
||||
}
|
||||
}()
|
||||
wf, ok := e.workflow(run.Workflow)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no workflow named %q", run.Workflow)
|
||||
}
|
||||
if run.Version != wf.version {
|
||||
return nil, fmt.Errorf("run started under version %d of workflow %q, this build runs version %d; start a new run",
|
||||
run.Version, run.Workflow, wf.version)
|
||||
wf, err := e.load(ctx, run)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
steps, err := e.work.Steps(ctx, run.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return wf.run(newContext(ctx, e.work, run, token, steps), run.Input)
|
||||
return wf(newContext(ctx, e.work, run, token, steps), run.Input)
|
||||
}
|
||||
|
||||
func (e *Engine) workflow(name string) (registered, bool) {
|
||||
e.mu.RLock()
|
||||
defer e.mu.RUnlock()
|
||||
wf, ok := e.workflows[name]
|
||||
return wf, ok
|
||||
// loadDocument reads the document a run executes: the version it started
|
||||
// under, or for a dry run the document it carries.
|
||||
func (e *Engine) loadDocument(ctx context.Context, run store.Run) (Workflow, error) {
|
||||
deps, err := e.steps()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body string
|
||||
if run.Dry {
|
||||
o, err := readOverlay(run.Overlay)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body = o.Document
|
||||
} else {
|
||||
w, err := e.db.Workflow(ctx, run.OrgID, run.Workflow, run.Version)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("workflow %q version %d: %w", run.Workflow, run.Version, err)
|
||||
}
|
||||
body = w.Body
|
||||
}
|
||||
doc, err := ParseDocument(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deps.Compile(doc), nil //nolint:contextcheck // a workflow takes its context from the run handle, as a handler takes it from the request
|
||||
}
|
||||
|
||||
func (e *Engine) keepLease(ctx context.Context, lost context.CancelFunc, id, token uuid.UUID) {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -133,7 +132,7 @@ func TestLeaseFencesStepWrites(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
engine := newEngine(nil, tt.work, slog.New(slog.DiscardHandler), Options{})
|
||||
effects := 0
|
||||
engine.Register("probe", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
useWorkflow(engine, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
out, err := rc.Step("probe", func(context.Context, string) (string, error) {
|
||||
return "mine", nil
|
||||
})
|
||||
@@ -178,7 +177,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
||||
}
|
||||
var seen []string
|
||||
register := func(e *Engine) {
|
||||
e.Register("pin", 1, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
useWorkflow(e, func(rc *Context, _ jsontext.Value) (jsontext.Value, error) {
|
||||
got, err := rc.Step("pin", func(context.Context, string) (pinned, error) {
|
||||
return pinned{Schema: jsontext.Value(`{"type": "object", "b": 1}`)}, nil
|
||||
})
|
||||
@@ -206,42 +205,7 @@ func TestStepValueIsTheSameLiveAndOnReplay(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunExecutesOnlyUnderItsStartVersion guards workflow versioning: a run
|
||||
// resumed after its workflow's version changed must fail naming both versions
|
||||
// and run none of the new code, never replay old steps through new logic.
|
||||
func TestRunExecutesOnlyUnderItsStartVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
runVersion int
|
||||
wantState string
|
||||
wantRan bool
|
||||
wantErrPart string
|
||||
}{
|
||||
{name: "same version runs", runVersion: 2, wantState: store.StateSucceeded, wantRan: true},
|
||||
{name: "older run fails naming both versions", runVersion: 1, wantState: store.StateFailed, wantErrPart: "version 1"},
|
||||
{name: "newer run fails when the build is behind", runVersion: 3, wantState: store.StateFailed, wantErrPart: "version 2"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
work := &fakeWork{}
|
||||
engine := newEngine(nil, work, slog.New(slog.DiscardHandler), Options{})
|
||||
ran := false
|
||||
engine.Register("versioned", 2, func(*Context, jsontext.Value) (jsontext.Value, error) {
|
||||
ran = true
|
||||
return jsontext.Value(`null`), nil
|
||||
})
|
||||
run := store.Run{ID: uuid.New(), OrgID: "org", Workflow: "versioned", Version: tt.runVersion, Input: jsontext.Value(`null`)}
|
||||
engine.execute(t.Context(), run, uuid.New())
|
||||
|
||||
if ran != tt.wantRan {
|
||||
t.Fatalf("workflow ran = %v, want %v", ran, tt.wantRan)
|
||||
}
|
||||
if len(work.finished) != 1 || work.finished[0] != tt.wantState {
|
||||
t.Fatalf("finished = %v, want [%s]", work.finished, tt.wantState)
|
||||
}
|
||||
if tt.wantErrPart != "" && !strings.Contains(work.errors[0], tt.wantErrPart) {
|
||||
t.Fatalf("error %q does not name %q", work.errors[0], tt.wantErrPart)
|
||||
}
|
||||
})
|
||||
}
|
||||
// useWorkflow makes e run wf for every run, in place of a stored document.
|
||||
func useWorkflow(e *Engine, wf Workflow) {
|
||||
e.load = func(context.Context, store.Run) (Workflow, error) { return wf, nil }
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
steps:
|
||||
- id: investigate
|
||||
agent:
|
||||
prompt: investigate
|
||||
memory: true
|
||||
- id: approve
|
||||
wait_for_approval: {}
|
||||
- id: tell
|
||||
notify: {}
|
||||
Reference in new issue
Block a user