feat(runs,approvals,ui): a run's trail joins each action's verdicts, approval and effect, and the run view shows them in order (archie #1996)
This commit is contained in:
7 files changed
+342
-18
No files matched your search
@@ -96,7 +96,7 @@ func Run(ctx context.Context, configPath string) error {
|
||||
// An answer a channel delivers is an event, and settles its request.
|
||||
hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow)
|
||||
hear.Routes(mux, guard)
|
||||
engine.Routes(mux, guard, orgID, hear, judge)
|
||||
engine.Routes(mux, guard, orgID, hear, judge, allow)
|
||||
dashboard.Routes(mux, ui.DistDirFS)
|
||||
|
||||
ctx, stop := context.WithCancel(ctx)
|
||||
|
||||
@@ -52,6 +52,8 @@ type Request struct {
|
||||
// Key is the run's wait key, stored so an answer knows what to wake.
|
||||
Key string
|
||||
Decision judgement.Decision
|
||||
DecisionID uuid.UUID
|
||||
RequestedAt time.Time
|
||||
Connector string
|
||||
Action string
|
||||
TargetKind string
|
||||
@@ -81,15 +83,20 @@ func (a Answer) Approval(d judgement.Decision) judgement.Approval {
|
||||
|
||||
// View is a request as the API shows it.
|
||||
type View struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
RunID uuid.UUID `json:"run_id"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
Level string `json:"level"`
|
||||
Summary string `json:"summary"`
|
||||
State string `json:"state"`
|
||||
Deadline time.Time `json:"deadline"`
|
||||
AnsweredBy string `json:"answered_by,omitzero"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
RunID uuid.UUID `json:"run_id"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
Level string `json:"level"`
|
||||
Summary string `json:"summary"`
|
||||
State string `json:"state"`
|
||||
Deadline time.Time `json:"deadline"`
|
||||
// DecisionID is the judged action the request is about.
|
||||
DecisionID uuid.UUID `json:"decision_id"`
|
||||
RequestedAt time.Time `json:"requested_at"`
|
||||
AnsweredBy string `json:"answered_by,omitzero"`
|
||||
AnsweredVia string `json:"answered_via,omitzero"`
|
||||
AnsweredAt time.Time `json:"answered_at,omitzero"`
|
||||
}
|
||||
|
||||
// View is r as the API shows it.
|
||||
@@ -97,7 +104,9 @@ func (r Request) View() View {
|
||||
return View{
|
||||
ID: r.ID, RunID: r.RunID, Action: r.Connector + " " + r.Action,
|
||||
Target: r.TargetKind + ":" + r.TargetID, Level: r.Level,
|
||||
Summary: r.Summary, State: r.State, Deadline: r.Deadline, AnsweredBy: r.AnsweredBy,
|
||||
Summary: r.Summary, State: r.State, Deadline: r.Deadline,
|
||||
DecisionID: r.DecisionID, RequestedAt: r.RequestedAt,
|
||||
AnsweredBy: r.AnsweredBy, AnsweredVia: r.AnsweredVia, AnsweredAt: r.AnsweredAt,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,6 +239,19 @@ func (m *Module) Pending(ctx context.Context) ([]Request, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ForRun returns a run's requests in the order they were made.
|
||||
func (m *Module) ForRun(ctx context.Context, runID uuid.UUID) ([]Request, error) {
|
||||
rows, err := m.store.ForRun(ctx, m.orgID, runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Request, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
out = append(out, requestOf(row, judgement.Decision{}))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Routes adds the approvals API to mux behind guard.
|
||||
func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) {
|
||||
mux.Handle("GET /api/approvals", guard(http.HandlerFunc(m.list)))
|
||||
@@ -361,7 +383,7 @@ func Summary(d judgement.Decision) string {
|
||||
// requestOf is a stored row as the module's Request.
|
||||
func requestOf(row store.Request, d judgement.Decision) Request {
|
||||
r := Request{
|
||||
ID: row.ID, RunID: row.RunID, Key: row.Key, Decision: d,
|
||||
ID: row.ID, RunID: row.RunID, Key: row.Key, Decision: d, DecisionID: row.DecisionID, RequestedAt: row.CreatedAt,
|
||||
Connector: row.Connector, Action: row.Action, TargetKind: row.TargetKind, TargetID: row.TargetID,
|
||||
Level: row.Level, Summary: row.Summary, State: row.State, Deadline: row.Deadline,
|
||||
}
|
||||
|
||||
@@ -125,5 +125,16 @@ func (s *Store) Pending(ctx context.Context, orgID string) ([]Request, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ForRun returns a run's requests, oldest first.
|
||||
func (s *Store) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]Request, error) {
|
||||
var out []Request
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT `+requestColumns+` FROM approvals.requests
|
||||
WHERE org_id = $1 AND run_id = $2 ORDER BY created_at, id`, orgID, runID); err != nil {
|
||||
return nil, fmt.Errorf("list run requests: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ErrNotFound means no request has that id or key.
|
||||
var ErrNotFound = errors.New("no such approval request")
|
||||
+141
-6
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/samcharles93/ai-sdk/chat"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/httpjson"
|
||||
"github.com/samcharles93/archie-core/modules/approvals"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
"github.com/samcharles93/archie-core/modules/runs/internal/store"
|
||||
)
|
||||
@@ -52,9 +53,39 @@ type RunView struct {
|
||||
type Trail struct {
|
||||
Run RunView `json:"run"`
|
||||
Origin *Origin `json:"origin,omitzero"`
|
||||
// Decisions are the judgements of the run's actions, with every verdict.
|
||||
Decisions []judgement.Decision `json:"decisions"`
|
||||
Entries []TrailEntry `json:"entries"`
|
||||
// Actions are the run's concrete actions in the order they were judged,
|
||||
// each with its verdicts, the approval it was put to and what came of it.
|
||||
Actions []ActionTrail `json:"actions"`
|
||||
Entries []TrailEntry `json:"entries"`
|
||||
}
|
||||
|
||||
// ActionTrail is one action from judgement to effect.
|
||||
type ActionTrail struct {
|
||||
Decision judgement.Decision `json:"decision"`
|
||||
// Rejudged is the judgement made after an approval, just before the action
|
||||
// ran.
|
||||
Rejudged *judgement.Decision `json:"rejudged,omitzero"`
|
||||
// Approval is the request put to a person, with the summary they were shown
|
||||
// and who answered. An action that needed none has none.
|
||||
Approval *approvals.View `json:"approval,omitzero"`
|
||||
// Effect is what the action did. An action that never ran has none.
|
||||
Effect *Effect `json:"effect,omitzero"`
|
||||
}
|
||||
|
||||
// The outcomes an action's effect can have.
|
||||
const (
|
||||
EffectRan = "ran"
|
||||
EffectFailed = "failed"
|
||||
EffectUnknown = "unknown"
|
||||
)
|
||||
|
||||
// Effect is what an action did: its output, its failure, or that it was
|
||||
// started and can't be confirmed.
|
||||
type Effect struct {
|
||||
State string `json:"state"`
|
||||
At time.Time `json:"at"`
|
||||
Output string `json:"output,omitzero"`
|
||||
Error string `json:"error,omitzero"`
|
||||
}
|
||||
|
||||
// TrailEntry is one recorded step. Kind is "model" or "compaction" for an
|
||||
@@ -87,9 +118,14 @@ type Decisions interface {
|
||||
ForRun(ctx context.Context, runID uuid.UUID) ([]judgement.Decision, error)
|
||||
}
|
||||
|
||||
// Approvals finds the approval requests made in a run.
|
||||
type Approvals interface {
|
||||
ForRun(ctx context.Context, runID uuid.UUID) ([]approvals.Request, error)
|
||||
}
|
||||
|
||||
// Routes adds the run API for orgID to mux behind guard.
|
||||
func (e *Engine) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler, orgID string, origins Origins, decisions Decisions) {
|
||||
api := &runAPI{db: e.db, orgID: orgID, origins: origins, decisions: decisions}
|
||||
func (e *Engine) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler, orgID string, origins Origins, decisions Decisions, asked Approvals) {
|
||||
api := &runAPI{db: e.db, orgID: orgID, origins: origins, decisions: decisions, approvals: asked}
|
||||
mux.Handle("GET /api/runs", guard(http.HandlerFunc(api.recent)))
|
||||
mux.Handle("GET /api/runs/{id}", guard(http.HandlerFunc(api.trail)))
|
||||
}
|
||||
@@ -99,6 +135,7 @@ type runAPI struct {
|
||||
orgID string
|
||||
origins Origins
|
||||
decisions Decisions
|
||||
approvals Approvals
|
||||
}
|
||||
|
||||
func (a *runAPI) recent(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -148,13 +185,111 @@ func (a *runAPI) trail(w http.ResponseWriter, r *http.Request) {
|
||||
for _, s := range steps {
|
||||
t.Entries = append(t.Entries, entry(s))
|
||||
}
|
||||
if t.Decisions, err = a.decisions.ForRun(r.Context(), id); err != nil {
|
||||
decisions, err := a.decisions.ForRun(r.Context(), id)
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusInternalServerError, "couldn't read the run's decisions")
|
||||
return
|
||||
}
|
||||
asked, err := a.approvals.ForRun(r.Context(), id)
|
||||
if err != nil {
|
||||
httpjson.Error(w, http.StatusInternalServerError, "couldn't read the run's approvals")
|
||||
return
|
||||
}
|
||||
t.Actions = actions(decisions, asked, steps)
|
||||
httpjson.Write(w, http.StatusOK, t)
|
||||
}
|
||||
|
||||
// actions joins each decision with its approval and its effect. An approved
|
||||
// action's effect is its run step, and the judgement made just before it is
|
||||
// folded into the action rather than listed as one of its own. An action that
|
||||
// ran unattended did so in the agent's tool call, whose journal position is the
|
||||
// decision's key.
|
||||
func actions(decisions []judgement.Decision, asked []approvals.Request, steps []store.Recorded) []ActionTrail {
|
||||
byName := make(map[string]store.Recorded, len(steps))
|
||||
tools := make(map[string]store.Recorded, len(steps))
|
||||
rejudgements := make(map[string]string, len(steps))
|
||||
for _, s := range steps {
|
||||
byName[s.Name] = s
|
||||
if key, ok := strings.CutPrefix(s.Name, "judge-again:"); ok {
|
||||
rejudgements[stepKey(s)] = key
|
||||
}
|
||||
}
|
||||
for _, s := range steps {
|
||||
if strings.HasPrefix(s.Name, "journal:agent:") {
|
||||
if key, ok := toolKey(s); ok {
|
||||
tools[key] = s
|
||||
}
|
||||
}
|
||||
}
|
||||
rejudged := make(map[string]judgement.Decision, len(rejudgements))
|
||||
out := make([]ActionTrail, 0, len(decisions))
|
||||
for _, d := range decisions {
|
||||
if of, ok := rejudgements[d.Key]; ok {
|
||||
rejudged[of] = d
|
||||
continue
|
||||
}
|
||||
out = append(out, ActionTrail{Decision: d})
|
||||
}
|
||||
for i := range out {
|
||||
d := out[i].Decision
|
||||
if r, ok := rejudged[d.Key]; ok {
|
||||
out[i].Rejudged = &r
|
||||
}
|
||||
for _, req := range asked {
|
||||
if req.DecisionID == d.ID {
|
||||
v := req.View()
|
||||
out[i].Approval = &v
|
||||
break
|
||||
}
|
||||
}
|
||||
switch run, ran := byName["run:"+d.Key]; {
|
||||
case ran:
|
||||
out[i].Effect = effectOf(run)
|
||||
case d.Level.Runs():
|
||||
if t, ok := tools[d.Key]; ok {
|
||||
out[i].Effect = effectOf(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// stepKey is the idempotency key the engine gave the step at s.
|
||||
func stepKey(s store.Recorded) string { return s.RunID.String() + ":" + strconv.Itoa(s.Seq) }
|
||||
|
||||
// toolKey is the idempotency key the agent loop gave the tool call a journal
|
||||
// step records.
|
||||
func toolKey(s store.Recorded) (string, bool) {
|
||||
var je agentloop.Entry
|
||||
if err := json.Unmarshal(s.Output, &je); err != nil || je.Kind != agentloop.EntryTool {
|
||||
return "", false
|
||||
}
|
||||
return s.RunID.String() + ":" + strconv.FormatInt(je.Position, 10), true
|
||||
}
|
||||
|
||||
// effectOf reads what an action's step recorded. A tool call's journal entry
|
||||
// and an approved action's run step record it differently.
|
||||
func effectOf(s store.Recorded) *Effect {
|
||||
e := TrailEntry{}
|
||||
if strings.HasPrefix(s.Name, "journal:agent:") {
|
||||
e = entry(s)
|
||||
} else {
|
||||
e.Error, e.Unknown = s.Error.String, s.Outcome == store.OutcomeUnknown
|
||||
var out string
|
||||
if err := json.Unmarshal(s.Output, &out); err == nil {
|
||||
e.Output = out
|
||||
}
|
||||
}
|
||||
ef := &Effect{State: EffectRan, At: s.RecordedAt, Output: e.Output, Error: e.Error}
|
||||
switch {
|
||||
case e.Unknown:
|
||||
ef.State = EffectUnknown
|
||||
case e.Error != "":
|
||||
ef.State = EffectFailed
|
||||
}
|
||||
return ef
|
||||
}
|
||||
|
||||
func view(s store.Summary, result jsontext.Value) RunView {
|
||||
v := RunView{
|
||||
ID: s.ID, Workflow: s.Workflow, State: s.State, Attempts: s.Attempts,
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from "vue";
|
||||
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { CodeBlock, CodeBlockContent } from "@/components/ui/code-block";
|
||||
import { MetadataList } from "@/components/ui/metadata-list";
|
||||
import { Timestamp } from "@/components/ui/timestamp";
|
||||
import type { ActionTrail, ApprovalRequest, Effect, Verdict } from "./api";
|
||||
|
||||
/** One action from judgement to effect: verdicts, the approval it was put to, and what it did. */
|
||||
const props = defineProps<{ action: ActionTrail }>();
|
||||
|
||||
type Tone = "ok" | "warn" | "danger" | "info" | "secondary";
|
||||
|
||||
const decision = computed(() => props.action.decision);
|
||||
const target = computed(() => `${decision.value.action.target.Kind}:${decision.value.action.target.ID}`);
|
||||
|
||||
const approvalTone: Record<ApprovalRequest["state"], Tone> = {
|
||||
pending: "warn",
|
||||
approved: "ok",
|
||||
denied: "danger",
|
||||
expired: "secondary",
|
||||
};
|
||||
const effectTone: Record<Effect["state"], Tone> = { ran: "ok", failed: "danger", unknown: "warn" };
|
||||
const effectLabel: Record<Effect["state"], string> = { ran: "Ran", failed: "Failed", unknown: "Outcome unknown" };
|
||||
|
||||
const approvalRows = computed(() => {
|
||||
const a = props.action.approval;
|
||||
if (!a?.answered_by) return [];
|
||||
return [
|
||||
{ label: "Answered by", value: a.answered_by },
|
||||
{ label: "Via", value: a.answered_via ?? "" },
|
||||
];
|
||||
});
|
||||
|
||||
const json = (v: unknown) => JSON.stringify(v, null, 2);
|
||||
const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary");
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="flex flex-col gap-3 rounded-md border p-3 text-label">
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<span class="font-medium">{{ decision.action.connector }} {{ decision.action.name }}</span>
|
||||
<span class="text-caption text-muted-foreground">{{ target }}</span>
|
||||
<Badge :variant="decision.reason ? 'danger' : 'secondary'">{{ decision.level }}</Badge>
|
||||
<Timestamp :at="decision.at" class="ml-auto text-caption" />
|
||||
</div>
|
||||
<p v-if="decision.action.effect.Summary" class="text-muted-foreground">{{ decision.action.effect.Summary }}</p>
|
||||
<CodeBlock size="sm">
|
||||
<CodeBlockContent wrap class="max-h-40">{{ json(decision.action.input) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
|
||||
<ul class="flex flex-col gap-1.5">
|
||||
<li v-for="v in decision.verdicts" :key="v.guard" class="flex flex-col gap-0.5">
|
||||
<span class="flex items-center gap-2">
|
||||
<span>{{ v.guard }}</span>
|
||||
<Badge :variant="ceilingTone(v)">{{ v.ceiling }}</Badge>
|
||||
<Badge v-if="v.advisory" variant="outline">advisory</Badge>
|
||||
</span>
|
||||
<span v-if="v.failed" class="text-caption text-destructive">{{ v.failed }}</span>
|
||||
<span v-for="f in v.findings" :key="f.text" class="text-caption text-muted-foreground">
|
||||
{{ f.text }}<template v-if="f.evidence">: {{ f.evidence }}</template>
|
||||
</span>
|
||||
<span v-for="b in v.blind_spots" :key="b" class="text-caption text-muted-foreground">Blind spot: {{ b }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
<p v-if="decision.reason" class="text-destructive">{{ decision.reason }}</p>
|
||||
|
||||
<div v-if="action.approval" class="flex flex-col gap-2 border-t pt-3">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="font-medium">Approval</span>
|
||||
<Badge :variant="approvalTone[action.approval.state]">{{ action.approval.state }}</Badge>
|
||||
<Timestamp :at="action.approval.answered_at ?? action.approval.requested_at" class="ml-auto text-caption" />
|
||||
</div>
|
||||
<MetadataList v-if="approvalRows.length" :rows="approvalRows" />
|
||||
<p class="whitespace-pre-wrap text-muted-foreground">{{ action.approval.summary }}</p>
|
||||
</div>
|
||||
|
||||
<div v-if="action.rejudged" class="flex items-center gap-2 text-caption text-muted-foreground">
|
||||
<span>Judged again</span>
|
||||
<Badge variant="secondary">{{ action.rejudged.level }}</Badge>
|
||||
<span v-if="action.rejudged.reason">{{ action.rejudged.reason }}</span>
|
||||
</div>
|
||||
|
||||
<div v-if="action.effect" class="flex flex-col gap-2 border-t pt-3">
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="font-medium">Effect</span>
|
||||
<Badge :variant="effectTone[action.effect.state]">{{ effectLabel[action.effect.state] }}</Badge>
|
||||
<Timestamp :at="action.effect.at" class="ml-auto text-caption" />
|
||||
</div>
|
||||
<p v-if="action.effect.error" class="text-destructive">{{ action.effect.error }}</p>
|
||||
<CodeBlock v-if="action.effect.output" size="sm">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ action.effect.output }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
TraceLine,
|
||||
TraceTitle,
|
||||
} from "@/components/ui/trace";
|
||||
import ActionTrailView from "./ActionTrailView.vue";
|
||||
import { runTrail, type Trail, type TrailEntry, Unauthorized } from "./api";
|
||||
import { runTone } from "./state";
|
||||
|
||||
@@ -103,6 +104,11 @@ const json = (v: unknown) => JSON.stringify(v, null, 2);
|
||||
<CodeBlockContent wrap class="max-h-64">{{ json(trail.origin.fields) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</Section>
|
||||
<Section v-if="trail.actions.length" title="Actions">
|
||||
<div class="flex flex-col gap-3">
|
||||
<ActionTrailView v-for="a in trail.actions" :key="a.decision.id" :action="a" />
|
||||
</div>
|
||||
</Section>
|
||||
<Section title="Steps">
|
||||
<Trace>
|
||||
<TraceCollapsible v-for="e in trail.entries" :key="e.seq" :default-open="Boolean(e.error || e.unknown)">
|
||||
|
||||
@@ -78,9 +78,62 @@ export interface TrailEntry {
|
||||
unknown?: boolean;
|
||||
}
|
||||
|
||||
export interface Verdict {
|
||||
guard: string;
|
||||
ceiling: string;
|
||||
findings?: { text: string; evidence?: string }[];
|
||||
blind_spots?: string[];
|
||||
advisory?: boolean;
|
||||
failed?: string;
|
||||
}
|
||||
|
||||
export interface Decision {
|
||||
id: string;
|
||||
key: string;
|
||||
action: {
|
||||
connector: string;
|
||||
name: string;
|
||||
input: unknown;
|
||||
digest: string;
|
||||
target: { Kind: string; ID: string; Environment: string };
|
||||
effect: { Summary: string };
|
||||
};
|
||||
level: string;
|
||||
verdicts: Verdict[];
|
||||
refused_by?: string;
|
||||
reason?: string;
|
||||
at: string;
|
||||
}
|
||||
|
||||
export interface ApprovalRequest {
|
||||
id: string;
|
||||
summary: string;
|
||||
state: "pending" | "approved" | "denied" | "expired";
|
||||
deadline: string;
|
||||
requested_at: string;
|
||||
answered_by?: string;
|
||||
answered_via?: string;
|
||||
answered_at?: string;
|
||||
}
|
||||
|
||||
export interface Effect {
|
||||
state: "ran" | "failed" | "unknown";
|
||||
at: string;
|
||||
output?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ActionTrail {
|
||||
decision: Decision;
|
||||
rejudged?: Decision;
|
||||
approval?: ApprovalRequest;
|
||||
effect?: Effect;
|
||||
}
|
||||
|
||||
export interface Trail {
|
||||
run: Run;
|
||||
origin?: Origin;
|
||||
actions: ActionTrail[];
|
||||
entries: TrailEntry[];
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user