421 lines
14 KiB
Go
421 lines
14 KiB
Go
package runs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/danielgtaylor/huma/v2"
|
|
"github.com/google/uuid"
|
|
"github.com/samcharles93/ai-sdk/agentloop"
|
|
"github.com/samcharles93/ai-sdk/chat"
|
|
|
|
"github.com/samcharles93/archie-core/internal/platform/httpapi"
|
|
"github.com/samcharles93/archie-core/modules/approvals"
|
|
"github.com/samcharles93/archie-core/modules/judgement"
|
|
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
|
)
|
|
|
|
// Origin is the event and trigger that started a run.
|
|
type Origin struct {
|
|
EventID uuid.UUID `json:"event_id"`
|
|
Source string `json:"source"`
|
|
Type string `json:"type"`
|
|
Fields jsontext.Value `json:"fields"`
|
|
ReceivedAt time.Time `json:"received_at"`
|
|
Trigger string `json:"trigger"`
|
|
Filter jsontext.Value `json:"filter"`
|
|
}
|
|
|
|
// Origins finds the event and trigger that started a run. A run started any
|
|
// other way has none.
|
|
type Origins interface {
|
|
Origin(ctx context.Context, runID uuid.UUID) (Origin, bool, error)
|
|
}
|
|
|
|
// RunView is a run as the API shows it.
|
|
type RunView struct {
|
|
ID uuid.UUID `json:"id"`
|
|
Workflow string `json:"workflow"`
|
|
Version int `json:"workflow_version"`
|
|
State string `json:"state"`
|
|
Attempts int `json:"attempts"`
|
|
Error string `json:"error,omitzero"`
|
|
// Detail explains how the run ended: a finish step's detail, or why a
|
|
// step parked the run.
|
|
Detail string `json:"detail,omitzero"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
FinishedAt *time.Time `json:"finished_at,omitzero"`
|
|
Result jsontext.Value `json:"result,omitzero"`
|
|
}
|
|
|
|
// Trail is a run's decision trail: what started it and every step it took,
|
|
// in order.
|
|
type Trail struct {
|
|
Run RunView `json:"run"`
|
|
Origin *Origin `json:"origin,omitzero"`
|
|
// Actions are the run's concrete actions in the order they were judged,
|
|
// each with its verdicts, the approval it was put to and what came of it.
|
|
Actions []ActionTrail `json:"actions"`
|
|
Entries []TrailEntry `json:"entries"`
|
|
}
|
|
|
|
// ActionTrail is one action from judgement to effect.
|
|
type ActionTrail struct {
|
|
Decision judgement.Decision `json:"decision"`
|
|
// Rejudged is the judgement made after an approval, just before the action
|
|
// ran.
|
|
Rejudged *judgement.Decision `json:"rejudged,omitzero"`
|
|
// Approval is the request put to a person, with the summary they were shown
|
|
// and who answered. An action that needed none has none.
|
|
Approval *approvals.View `json:"approval,omitzero"`
|
|
// Effect is what the action did. An action that never ran has none.
|
|
Effect *Effect `json:"effect,omitzero"`
|
|
}
|
|
|
|
// The outcomes an action's effect can have.
|
|
const (
|
|
EffectRan = "ran"
|
|
EffectFailed = "failed"
|
|
EffectUnknown = "unknown"
|
|
)
|
|
|
|
// Effect is what an action did: its output, its failure, or that it was
|
|
// started and can't be confirmed.
|
|
type Effect struct {
|
|
State string `json:"state"`
|
|
At time.Time `json:"at"`
|
|
Output string `json:"output,omitzero"`
|
|
Error string `json:"error,omitzero"`
|
|
}
|
|
|
|
// TrailEntry is one recorded step. Kind is "model" or "compaction" for an
|
|
// agent's model call, "tool" for its tool call, and "step" for any other
|
|
// workflow step.
|
|
type TrailEntry struct {
|
|
Seq int `json:"seq"`
|
|
At time.Time `json:"at"`
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name,omitzero"`
|
|
Text string `json:"text,omitzero"`
|
|
Reasoning string `json:"reasoning,omitzero"`
|
|
ToolCalls []ToolCallView `json:"tool_calls,omitzero"`
|
|
Usage *chat.Usage `json:"usage,omitzero"`
|
|
// Request is the model request exactly as it was sent: the messages, the
|
|
// tools offered and the sampling settings.
|
|
Request jsontext.Value `json:"request,omitzero"`
|
|
// FinishReason is the wire finish reason of a model call.
|
|
FinishReason string `json:"finish_reason,omitzero"`
|
|
Input string `json:"input,omitzero"`
|
|
Output string `json:"output,omitzero"`
|
|
Value jsontext.Value `json:"value,omitzero"`
|
|
Error string `json:"error,omitzero"`
|
|
Unknown bool `json:"unknown,omitzero"`
|
|
}
|
|
|
|
// ToolCallView is a tool call a model asked for.
|
|
type ToolCallView struct {
|
|
Name string `json:"name"`
|
|
Input string `json:"input"`
|
|
}
|
|
|
|
// Decisions finds the judgements made in a run.
|
|
type Decisions interface {
|
|
ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error)
|
|
}
|
|
|
|
// Approvals finds the approval requests made in a run.
|
|
type Approvals interface {
|
|
ForRun(ctx context.Context, runID uuid.UUID) ([]approvals.Request, error)
|
|
}
|
|
|
|
// Register adds the run API for orgID: recent runs, each run's trail, and
|
|
// the workflows runs execute.
|
|
func (e *Engine) Register(api huma.API, orgID string, origins Origins, decisions Decisions, asked Approvals) {
|
|
a := &runAPI{engine: e, db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
|
|
tags := []string{"Runs"}
|
|
huma.Register(api, huma.Operation{
|
|
OperationID: "list-runs", Method: http.MethodGet, Path: "/api/runs",
|
|
Summary: "List recent runs", Tags: tags,
|
|
}, a.recent)
|
|
huma.Register(api, huma.Operation{
|
|
OperationID: "get-run", Method: http.MethodGet, Path: "/api/runs/{id}",
|
|
Summary: "Get a run's decision trail", Tags: tags,
|
|
}, a.trail)
|
|
huma.Register(api, huma.Operation{
|
|
OperationID: "retry-run", Method: http.MethodPost, Path: "/api/runs/{id}/retry",
|
|
Summary: "Retry a parked run from a step, keeping the results of the steps before it", Tags: tags,
|
|
}, a.retry)
|
|
e.registerWorkflows(api, orgID)
|
|
}
|
|
|
|
type retryRunInput struct {
|
|
ID string `path:"id" format:"uuid"`
|
|
Body struct {
|
|
From string `json:"from,omitempty" doc:"The step to retry from, by id or as the trail names it; by default the step that parked the run"`
|
|
}
|
|
}
|
|
|
|
func (a *runAPI) retry(ctx context.Context, in *retryRunInput) (*httpapi.Reply[RunView], error) {
|
|
id, err := uuid.Parse(in.ID)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusBadRequest, "a run id is a UUID")
|
|
}
|
|
switch err := a.engine.Retry(ctx, a.orgID, id, in.Body.From); {
|
|
case errors.Is(err, store.ErrNoRun):
|
|
return nil, huma.NewError(http.StatusNotFound, "no such run")
|
|
case errors.Is(err, ErrNotRetryable):
|
|
return nil, huma.NewError(http.StatusConflict, "only a parked run retries, and only from a step it reached")
|
|
case err != nil:
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't retry the run")
|
|
}
|
|
d, ok, err := a.db.Detail(ctx, a.orgID, id)
|
|
if err != nil || !ok {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run")
|
|
}
|
|
return httpapi.OK(view(d.Summary, d.Result)), nil
|
|
}
|
|
|
|
type recentRunsInput struct {
|
|
Limit int `query:"limit" minimum:"1" maximum:"200" default:"50"`
|
|
}
|
|
|
|
type runInput struct {
|
|
ID string `path:"id" format:"uuid"`
|
|
}
|
|
|
|
type runAPI struct {
|
|
engine *Engine
|
|
db *store.Store
|
|
orgID string
|
|
origins Origins
|
|
decisions Decisions
|
|
approvals Approvals
|
|
}
|
|
|
|
func (a *runAPI) recent(ctx context.Context, in *recentRunsInput) (*httpapi.Reply[[]RunView], error) {
|
|
runs, err := a.db.Recent(ctx, a.orgID, in.Limit)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't list runs")
|
|
}
|
|
views := make([]RunView, 0, len(runs))
|
|
for _, s := range runs {
|
|
views = append(views, view(s, nil))
|
|
}
|
|
return httpapi.OK(views), nil
|
|
}
|
|
|
|
func (a *runAPI) trail(ctx context.Context, in *runInput) (*httpapi.Reply[Trail], error) {
|
|
id, err := uuid.Parse(in.ID)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusBadRequest, "a run id is a UUID")
|
|
}
|
|
d, ok, err := a.db.Detail(ctx, a.orgID, id)
|
|
switch {
|
|
case err != nil:
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run")
|
|
case !ok:
|
|
return nil, huma.NewError(http.StatusNotFound, "no such run")
|
|
}
|
|
steps, err := a.db.RecordedSteps(ctx, id)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run's steps")
|
|
}
|
|
t := Trail{Run: view(d.Summary, d.Result), Entries: make([]TrailEntry, 0, len(steps))}
|
|
if o, ok, err := a.origins.Origin(ctx, id); err != nil {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read what started the run")
|
|
} else if ok {
|
|
t.Origin = &o
|
|
}
|
|
for _, s := range steps {
|
|
t.Entries = append(t.Entries, entry(s))
|
|
}
|
|
decisions, err := a.decisions.ForRun(ctx, id)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run's decisions")
|
|
}
|
|
asked, err := a.approvals.ForRun(ctx, id)
|
|
if err != nil {
|
|
return nil, huma.NewError(http.StatusInternalServerError, "couldn't read the run's approvals")
|
|
}
|
|
t.Actions = actions(decisions, asked, steps)
|
|
return httpapi.OK(t), nil
|
|
}
|
|
|
|
// actions joins each decision with its approval and its effect. An approved
|
|
// action's effect is its run step, and the judgement made just before it is
|
|
// folded into the action rather than listed as one of its own. An action that
|
|
// ran unattended did so in the agent's tool call, whose journal position is the
|
|
// decision's key.
|
|
func actions(decisions []judgement.Decision, asked []approvals.Request, steps []store.Recorded) []ActionTrail {
|
|
byName := make(map[string]store.Recorded, len(steps))
|
|
tools := make(map[string]store.Recorded, len(steps))
|
|
rejudgements := make(map[string]string, len(steps))
|
|
for _, s := range steps {
|
|
byName[s.Name] = s
|
|
if key, ok := strings.CutPrefix(s.Name, "judge-again:"); ok {
|
|
rejudgements[stepKey(s)] = key
|
|
}
|
|
}
|
|
for _, s := range steps {
|
|
if strings.HasPrefix(s.Name, "journal:agent:") {
|
|
if key, ok := toolKey(s); ok {
|
|
tools[key] = s
|
|
}
|
|
}
|
|
}
|
|
rejudged := make(map[string]judgement.Decision, len(rejudgements))
|
|
out := make([]ActionTrail, 0, len(decisions))
|
|
for _, d := range decisions {
|
|
if of, ok := rejudgements[d.Key]; ok {
|
|
rejudged[of] = d
|
|
continue
|
|
}
|
|
out = append(out, ActionTrail{Decision: d})
|
|
}
|
|
for i := range out {
|
|
d := out[i].Decision
|
|
if r, ok := rejudged[d.Key]; ok {
|
|
out[i].Rejudged = &r
|
|
}
|
|
for _, req := range asked {
|
|
if req.DecisionID == d.ID {
|
|
v := req.View()
|
|
out[i].Approval = &v
|
|
break
|
|
}
|
|
}
|
|
switch run, ran := byName["run:"+d.Key]; {
|
|
case ran:
|
|
out[i].Effect = effectOf(run)
|
|
case d.Level.Runs():
|
|
if t, ok := tools[d.Key]; ok {
|
|
out[i].Effect = effectOf(t)
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// stepKey is the idempotency key the engine gave the step at s.
|
|
func stepKey(s store.Recorded) string { return s.RunID.String() + ":" + strconv.Itoa(s.Seq) }
|
|
|
|
// toolKey is the idempotency key the agent loop gave the tool call a journal
|
|
// step records.
|
|
func toolKey(s store.Recorded) (string, bool) {
|
|
agent, ok := strings.CutPrefix(s.Name, "journal:agent:")
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
var je agentloop.Entry
|
|
if err := json.Unmarshal(s.Output, &je); err != nil || je.Kind != agentloop.EntryTool {
|
|
return "", false
|
|
}
|
|
return s.RunID.String() + ":" + agent + ":" + strconv.FormatInt(je.Position, 10), true
|
|
}
|
|
|
|
// effectOf reads what an action's step recorded. A tool call's journal entry
|
|
// and an approved action's run step record it differently.
|
|
func effectOf(s store.Recorded) *Effect {
|
|
e := TrailEntry{}
|
|
if strings.HasPrefix(s.Name, "journal:agent:") {
|
|
e = entry(s)
|
|
} else {
|
|
e.Error, e.Unknown = s.Error.String, s.Outcome == store.OutcomeUnknown
|
|
var out string
|
|
if err := json.Unmarshal(s.Output, &out); err == nil {
|
|
e.Output = out
|
|
}
|
|
}
|
|
ef := &Effect{State: EffectRan, At: s.RecordedAt, Output: e.Output, Error: e.Error}
|
|
switch {
|
|
case e.Unknown:
|
|
ef.State = EffectUnknown
|
|
case e.Error != "":
|
|
ef.State = EffectFailed
|
|
}
|
|
return ef
|
|
}
|
|
|
|
func view(s store.Summary, result jsontext.Value) RunView {
|
|
v := RunView{
|
|
ID: s.ID, Workflow: s.Workflow, Version: s.Version, State: s.State, Attempts: s.Attempts,
|
|
Error: s.Error.String, Detail: s.Detail.String, CreatedAt: s.CreatedAt, Result: result,
|
|
}
|
|
if s.FinishedAt.Valid {
|
|
v.FinishedAt = &s.FinishedAt.Time
|
|
}
|
|
return v
|
|
}
|
|
|
|
// entry turns a recorded step into a trail entry. An agent's journal steps
|
|
// are unpacked into the model and tool calls they record.
|
|
func entry(s store.Recorded) TrailEntry {
|
|
e := TrailEntry{Seq: s.Seq, At: s.RecordedAt, Kind: "step", Name: s.Name, Error: s.Error.String}
|
|
if s.Outcome == store.OutcomeUnknown {
|
|
e.Unknown = true
|
|
}
|
|
if !strings.HasPrefix(s.Name, "journal:agent:") {
|
|
e.Value = s.Output
|
|
return e
|
|
}
|
|
var je agentloop.Entry
|
|
if err := json.Unmarshal(s.Output, &je); err != nil {
|
|
e.Value = s.Output
|
|
return e
|
|
}
|
|
switch je.Kind {
|
|
case agentloop.EntryTool:
|
|
e.Kind, e.Name = "tool", je.ToolName
|
|
e.Input, e.Output, e.Error, e.Unknown = je.Input, je.Output, je.Error, je.Unknown
|
|
case agentloop.EntryModel, agentloop.EntryCompact:
|
|
e.Kind, e.Name = "model", ""
|
|
if je.Kind == agentloop.EntryCompact {
|
|
e.Kind = "compaction"
|
|
}
|
|
e.Request, e.FinishReason = je.Request, je.FinishReason
|
|
usage := je.Usage
|
|
e.Usage = &usage
|
|
if je.Message != nil {
|
|
e.Text = je.Message.Content
|
|
for _, p := range je.Message.Parts {
|
|
if rp, ok := p.(chat.ReasoningPart); ok {
|
|
e.Reasoning += rp.Text
|
|
}
|
|
}
|
|
for _, c := range je.Message.ToolCalls {
|
|
e.ToolCalls = append(e.ToolCalls, ToolCallView{Name: c.Name, Input: c.Arguments})
|
|
}
|
|
}
|
|
e.Error = je.Error
|
|
}
|
|
return e
|
|
}
|
|
|
|
// Recent returns orgID's latest runs, newest first.
|
|
func (e *Engine) Recent(ctx context.Context, orgID string, limit int) ([]RunView, error) {
|
|
runs, err := e.db.Recent(ctx, orgID, limit)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]RunView, 0, len(runs))
|
|
for _, s := range runs {
|
|
out = append(out, view(s, nil))
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Run returns one run with its result; ok is false when there is no such run.
|
|
func (e *Engine) Run(ctx context.Context, orgID string, id uuid.UUID) (RunView, bool, error) {
|
|
d, ok, err := e.db.Detail(ctx, orgID, id)
|
|
if err != nil || !ok {
|
|
return RunView{}, ok, err
|
|
}
|
|
return view(d.Summary, d.Result), true, nil
|
|
}
|