Files
sam 8c267b6edd feat: record skipped dispatches and approver ids
- store trigger dispatches as started or skipped, with a JSON reason surfaced in the API and UI
- persist the answering account's identity id beside its display name for approvals and judgements
- save connector acceptances and their declarations in one transaction so a refused change stores nothing
2026-11-22 23:24:30 +11:00

518 lines
19 KiB
Go

// Package judgement decides whether a concrete action runs, and runs only the
// action it judged. Hard rules refuse first; every guard then sets a ceiling,
// and the action runs at the lowest one. Each decision is recorded with every
// verdict, so the run's trail and the action class's history can read it.
package judgement
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"log/slog"
"slices"
"sync"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/judgement/internal/store"
)
// guardTimeout bounds one guard, so a slow guard holds an action no longer.
const guardTimeout = 10 * time.Second
var (
// ErrRefused means a hard rule refused the action.
ErrRefused = errors.New("refused by a hard rule")
// ErrHeld means the action's level needs someone's approval first.
ErrHeld = errors.New("held for approval")
// ErrAltered means the action to run isn't the one that was judged.
ErrAltered = errors.New("action differs from the one judged")
// ErrDryRun means the decision was made for a dry run and the action
// changes something, so it is never executed.
ErrDryRun = errors.New("dry run: not executed")
// ErrUnapproved means the action needs an approval it doesn't have, or the
// approval offered was given for a different action.
ErrUnapproved = errors.New("not approved")
)
// Action is one concrete action: the exact operation on the exact target.
type Action struct {
Connector string `json:"connector"`
Name string `json:"name"`
Input jsontext.Value `json:"input"`
Digest string `json:"digest"`
Target connectors.Target `json:"target"`
Effect connectors.Effect `json:"effect"`
}
// Class is what an action's earned autonomy belongs to.
type Class struct {
Connector string `json:"connector"`
Action string `json:"action"`
TargetKind string `json:"target_kind"`
Environment string `json:"environment"`
}
// Class returns the action's class.
func (a Action) Class() Class {
return Class{Connector: a.Connector, Action: a.Name, TargetKind: a.Target.Kind, Environment: a.Target.Environment}
}
// Finding is one thing a guard noticed, with what it rests on.
type Finding struct {
Text string `json:"text"`
Evidence string `json:"evidence,omitzero"`
// Strength is how much the evidence supports the finding: weak, moderate
// or strong.
Strength string `json:"strength,omitzero"`
}
// JudgeRecord is what one model judge answered and what it was judged on,
// kept whole so the verdict can be audited without asking the model again.
type JudgeRecord struct {
Model string `json:"model,omitzero"`
ResultID string `json:"result_id,omitzero"`
Answer string `json:"answer,omitzero"`
Confidence float64 `json:"confidence,omitzero"`
Reason string `json:"reason,omitzero"`
// Basis is the text the judge was given beyond the action, and
// BasisDigest its SHA-256.
Basis string `json:"basis,omitzero"`
BasisDigest string `json:"basis_digest,omitzero"`
// Answers is the judge service's answers as it returned them.
Answers jsontext.Value `json:"answers,omitzero"`
// Trusted says the judge has passed its eval; AdvisoryReason says why
// not when it hasn't.
Trusted bool `json:"trusted,omitzero"`
AdvisoryReason string `json:"advisory_reason,omitzero"`
EvalDate string `json:"eval_date,omitzero"`
// Request and Response are the exact bodies the judge endpoint was sent
// and returned, kept for audit.
Request jsontext.Value `json:"request,omitzero"`
Response jsontext.Value `json:"response,omitzero"`
EndpointPath string `json:"endpoint_path,omitzero"`
HTTPStatus int `json:"http_status,omitzero"`
LatencyMS int64 `json:"latency_ms,omitzero"`
}
// Verdict is one guard's judgement of one action.
type Verdict struct {
Guard string `json:"guard"`
// Version and Digest say which extension answered: its own release and
// the digest of the package that ran. Built-in guards have neither.
Version string `json:"version,omitzero"`
Digest string `json:"digest,omitzero"`
Ceiling Level `json:"ceiling"`
Findings []Finding `json:"findings,omitzero"`
BlindSpots []string `json:"blind_spots,omitzero"`
// Judges is every model judgement the verdict rests on.
Judges []JudgeRecord `json:"judges,omitzero"`
// Advisory verdicts are recorded and set no ceiling.
Advisory bool `json:"advisory,omitzero"`
// Failed says why the guard couldn't answer; its ceiling is then
// propose and wait.
Failed string `json:"failed,omitzero"`
// Owner is the id of the person the impact guard found answering for the
// target, empty when it found none.
Owner string `json:"owner,omitzero"`
}
// Guard judges one concrete action.
type Guard interface {
Name() string
// Advisory guards are recorded but set no ceiling.
Advisory() bool
// Judge returns the guard's verdict. An error, a timeout or a verdict
// with no ceiling holds the action at propose and wait.
Judge(ctx context.Context, a Action) (Verdict, error)
}
// HardRule refuses actions outright. Hard rules run before any guard.
type HardRule interface {
// Refuses reports whether a rule forbids the action: the rule's id and why.
Refuses(ctx context.Context, a Action) (by, reason string, refused bool)
}
// Decision is the judgement of one action.
type Decision struct {
ID uuid.UUID `json:"id"`
RunID uuid.UUID `json:"run_id"`
Key string `json:"key"`
Action Action `json:"action"`
Level Level `json:"level"`
Verdicts []Verdict `json:"verdicts"`
// RefusedBy names the hard rule that refused the action, if one did.
RefusedBy string `json:"refused_by,omitzero"`
// Workflow is the workflow the decision was made for. Execution checks it
// again, so a held chat action stays prohibited whatever approval it has.
Workflow string `json:"workflow,omitzero"`
// ChangeRequest says approval goes through a change request, and
// ChangeRequestRule which change-request rule said so.
ChangeRequest bool `json:"change_request,omitzero"`
ChangeRequestRule string `json:"change_request_rule,omitzero"`
Reason string `json:"reason,omitzero"`
At time.Time `json:"at"`
// Dry marks a decision made for a dry run. It is recorded for the run's
// trail and counts toward nothing: not spend, not a class's history.
Dry bool `json:"dry,omitzero"`
// Calls are the calls to the connector behind this decision: the resolve
// that placed its target, the target facts it was judged on, and the
// invoke that ran it. Each holds the request core sent and the response it
// received, with credentials removed.
Calls []connectors.Call `json:"calls,omitzero"`
}
// Owner is the id of the person who answers for the decision's target, or empty
// when the impact guard found none. Only the built-in guard's word counts, so
// an extension can't name who is asked.
func (d Decision) Owner() string {
for _, v := range d.Verdicts {
if v.Guard == (Impact{}).Name() && v.Owner != "" {
return v.Owner
}
}
return ""
}
// Request asks for one action to be judged. Key identifies the attempt to run
// it: the same key is the same decision, so a replayed step never judges
// twice.
type Request struct {
RunID uuid.UUID
Key string
Connector string
Action string
Input jsontext.Value
// Workflow is the workflow the run executes, the principal hard rules see.
Workflow string
// Diagnosis and Evidence are what the proposing agent concluded and read,
// for guards that weigh the reasoning behind an action. Both may be empty.
Diagnosis string
Evidence []Evidence
// Dry judges the action for a dry run: the decision is recorded apart from
// the history autonomy and budgets count, and it is never executed unless
// the action only reads.
Dry bool
// Overlay is policy text a dry run judges under as if it were published.
Overlay *PolicyOverlay
}
// Tools finds the tool that resolves and runs an action.
type Tools interface {
Tool(name string) (connectors.Tool, bool)
}
// FailureRecorder is told when an action of a class failed, so the class loses
// the autonomy it had.
type FailureRecorder interface {
Failed(ctx context.Context, c Class) error
}
type recorder interface {
Record(ctx context.Context, d store.Decision) (store.Decision, error)
ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]store.Decision, error)
AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error
}
// callCollector keeps the connector calls one judgement made, so the decision
// body records what core sent and received.
type callCollector struct {
mu sync.Mutex
calls []connectors.Call
}
// RecordConnectorCall keeps one call. Guards judge in parallel, so the
// collector is safe for concurrent use.
func (c *callCollector) RecordConnectorCall(_ context.Context, call connectors.Call) {
c.mu.Lock()
defer c.mu.Unlock()
c.calls = append(c.calls, call)
}
// take returns the calls collected so far.
func (c *callCollector) take() []connectors.Call {
c.mu.Lock()
defer c.mu.Unlock()
return slices.Clone(c.calls)
}
// Module judges one organisation's actions.
type Module struct {
orgID string
tools Tools
rules []HardRule
guards []Guard
store recorder
// queries reads the decisions a guard's last judgement is found in.
queries *store.Store
failed FailureRecorder
// changeRules decides whether approval goes through a change request.
changeRules ChangeRules
// extensions are the guard extensions asked alongside the built-in guards.
extensions GuardExtensions
// timeout bounds one guard; zero means guardTimeout.
timeout time.Duration
log *slog.Logger
now func() time.Time
}
// UseChangeRules makes the module ask rules whether a held action's approval
// goes through a change request.
func (m *Module) UseChangeRules(r ChangeRules) { m.changeRules = r }
// UseExtensions makes the module ask every running guard extension alongside
// its built-in guards, on each decision.
func (m *Module) UseExtensions(e GuardExtensions) { m.extensions = e }
// NewModule returns the judgement module for orgID, with its hard rules and
// guards. An action of a class that fails is reported to failed.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, tools Tools, rules []HardRule, guards []Guard, failed FailureRecorder) *Module {
s := store.New(db)
return &Module{orgID: orgID, tools: tools, rules: rules, guards: guards, store: s, queries: s, failed: failed, log: log, now: time.Now}
}
// Assess judges one action and records the decision. Judging the same key
// again returns the recorded decision.
func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) {
tool, ok := m.tools.Tool(req.Connector)
if !ok {
return Decision{}, fmt.Errorf("%q isn't running", req.Connector)
}
calls := &callCollector{}
ctx = connectors.WithRecorder(ctx, calls)
res, err := tool.Resolve(ctx, req.Action, req.Input)
if err != nil {
return Decision{}, fmt.Errorf("resolve %s %s: %w", req.Connector, req.Action, err)
}
a := Action{
Connector: req.Connector, Name: req.Action, Input: req.Input,
Digest: connectors.Digest(req.Action, req.Input), Target: res.Target, Effect: res.Effect,
}
d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence, Overlay: req.Overlay}), a)
d.ID, d.RunID, d.Key, d.At, d.Dry = uuid.New(), req.RunID, req.Key, m.now().UTC(), req.Dry
d.Workflow = req.Workflow
d.Calls = calls.take()
return m.record(ctx, d)
}
// decide runs the hard rules, then every guard in parallel, and folds their
// ceilings: the action runs at the lowest.
func (m *Module) decide(ctx context.Context, a Action) Decision {
if reason, prohibited := chatProhibition(runFrom(ctx).Workflow, a); prohibited {
return chatRefusal(a, reason)
}
for _, r := range m.rules {
if by, reason, refused := r.Refuses(ctx, a); refused {
return Decision{Action: a, Level: Never, RefusedBy: by, Reason: reason, Verdicts: []Verdict{}}
}
}
guards := slices.Clone(m.guards)
if m.extensions != nil {
for _, e := range m.extensions.Guards() {
guards = append(guards, extensionGuard{e})
}
}
verdicts := make([]Verdict, len(guards))
var wg sync.WaitGroup
for i, g := range guards {
wg.Go(func() { verdicts[i] = judge(ctx, g, a, m.guardTimeout()) })
}
wg.Wait()
ceilings := []Level{}
for _, v := range verdicts {
if !v.Advisory {
ceilings = append(ceilings, v.Ceiling)
}
}
level := ProposeAndWait
if len(ceilings) > 0 {
level = lowest(ceilings...)
}
d := Decision{Action: a, Level: level, Verdicts: verdicts}
// A change request is a way to approve, so only a held action has one.
if m.changeRules != nil && level > ActAndTell && level < Never {
d.ChangeRequestRule, d.ChangeRequest = m.changeRules.Requires(ctx, a, level, verdicts)
}
return d
}
// judge asks one guard, turning a failure, a timeout or a missing ceiling
// into propose and wait. A panic is a failure too.
func judge(ctx context.Context, g Guard, a Action, timeout time.Duration) (v Verdict) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
failed := func(why string) Verdict {
v := Verdict{Guard: g.Name(), Ceiling: ProposeAndWait, Advisory: g.Advisory(), Failed: why}
if e, ok := g.(identified); ok {
v.Version, v.Digest = e.Version(), e.Digest()
}
return v
}
defer func() {
if r := recover(); r != nil {
v = failed(fmt.Sprintf("panicked: %v", r))
}
}()
got, err := g.Judge(ctx, a)
switch {
case err != nil:
return failed(err.Error())
case got.Ceiling < ActQuietly || got.Ceiling > Never:
return failed("returned no ceiling")
}
got.Guard, got.Advisory = g.Name(), g.Advisory()
if e, ok := g.(identified); ok {
got.Version, got.Digest = e.Version(), e.Digest()
}
return got
}
func (m *Module) guardTimeout() time.Duration {
if m.timeout > 0 {
return m.timeout
}
return guardTimeout
}
// Execute runs a decided action if its level lets it run and the input is
// exactly the one judged. key is the idempotency key the connector honours.
func (m *Module) Execute(ctx context.Context, d Decision, input jsontext.Value, key string) (string, error) {
if reason, prohibited := chatProhibition(d.Workflow, d.Action); prohibited {
return "", fmt.Errorf("%w: %s: %s", ErrRefused, chatProhibitionRule, reason)
}
switch {
case d.Level == Never:
return "", fmt.Errorf("%w: %s: %s", ErrRefused, d.RefusedBy, d.Reason)
case !d.Level.Runs():
return "", fmt.Errorf("%w at %s", ErrHeld, d.Level)
case connectors.Digest(d.Action.Name, input) != d.Action.Digest:
return "", ErrAltered
case d.Dry && !d.Action.Effect.ReadOnly:
return "", ErrDryRun
}
return m.invoke(ctx, d, input, key)
}
// invoke runs the judged action. A failure, or an outcome nobody can confirm,
// drops the action's class to propose and wait at once, whatever level it ran at.
func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, key string) (string, error) {
tool, ok := m.tools.Tool(d.Action.Connector)
if !ok {
return "", fmt.Errorf("%q isn't running", d.Action.Connector)
}
calls := &callCollector{}
ctx = connectors.WithRecorder(ctx, calls)
out, err := tool.Invoke(ctx, connectors.Invocation{Action: d.Action.Name, Input: input, IdempotencyKey: key})
m.keepCalls(ctx, d.ID, calls.take())
// A shutdown cancels the call without the action having gone wrong, and a
// dry run's failure is not evidence about the class.
if err != nil && m.failed != nil && !d.Dry && !errors.Is(err, context.Canceled) {
if ferr := m.failed.Failed(ctx, d.Action.Class()); ferr != nil {
return out, errors.Join(err, fmt.Errorf("tighten class: %w", ferr))
}
}
return out, err
}
// keepCalls adds the connector calls one invoke made to its decision body, so
// the executed action's request and response sit with the judgement that let
// it run.
func (m *Module) keepCalls(ctx context.Context, id uuid.UUID, calls []connectors.Call) {
if id == uuid.Nil {
return
}
for _, call := range calls {
raw, err := json.Marshal(call)
if err != nil {
m.log.ErrorContext(ctx, "encode connector call", "decision", id, "error", err)
continue
}
if err := m.store.AppendCall(ctx, m.orgID, id, string(raw)); err != nil {
m.log.ErrorContext(ctx, "record connector call", "decision", id, "error", err)
}
}
}
// Approval is a person's permission to run one decision's exact action. It
// names the decision and the action's digest, so it covers that action and
// nothing else, and it says who gave it and when.
type Approval struct {
DecisionID uuid.UUID `json:"decision_id"`
Digest string `json:"digest"`
By string `json:"by"`
// ByID is the identity ID of the account named by By.
ByID string `json:"by_id,omitzero"`
At time.Time `json:"at"`
}
// Covers reports whether the approval was given for d's action.
func (a Approval) Covers(d Decision) bool {
return a.DecisionID == d.ID && a.Digest == d.Action.Digest
}
// ExecuteApproved runs an action a person approved. A refusal still refuses,
// and an approval that names another decision or another action runs nothing.
// The action that runs is the one judged, whatever input comes with it.
func (m *Module) ExecuteApproved(ctx context.Context, d Decision, a Approval, input jsontext.Value, key string) (string, error) {
if reason, prohibited := chatProhibition(d.Workflow, d.Action); prohibited {
return "", fmt.Errorf("%w: %s: %s", ErrRefused, chatProhibitionRule, reason)
}
switch {
case d.Level == Never:
return "", fmt.Errorf("%w: %s: %s", ErrRefused, d.RefusedBy, d.Reason)
case !a.Covers(d):
return "", fmt.Errorf("%w: the approval names another action", ErrUnapproved)
case connectors.Digest(d.Action.Name, input) != d.Action.Digest:
return "", ErrAltered
case d.Dry && !d.Action.Effect.ReadOnly:
return "", ErrDryRun
}
return m.invoke(ctx, d, input, key)
}
// ForRun returns a run's decisions in the order they were made.
func (m *Module) ForRun(ctx context.Context, runID uuid.UUID) ([]Decision, error) {
rows, err := m.store.ForRun(ctx, m.orgID, runID)
if err != nil {
return nil, err
}
out := make([]Decision, 0, len(rows))
for _, r := range rows {
var d Decision
if err := json.Unmarshal(r.Body, &d); err != nil {
return nil, fmt.Errorf("decision %s: %w", r.ID, err)
}
out = append(out, d)
}
return out, nil
}
func (m *Module) record(ctx context.Context, d Decision) (Decision, error) {
body, err := json.Marshal(d)
if err != nil {
return Decision{}, fmt.Errorf("encode decision: %w", err)
}
c := d.Action.Class()
row, err := m.store.Record(ctx, store.Decision{
ID: d.ID, OrgID: m.orgID, RunID: d.RunID, Key: d.Key,
Connector: c.Connector, Action: c.Action, TargetKind: c.TargetKind, Environment: c.Environment,
Level: d.Level.String(), Dry: d.Dry, Body: body,
})
if err != nil {
return Decision{}, err
}
if row.ID == d.ID {
return d, nil
}
var kept Decision
if err := json.Unmarshal(row.Body, &kept); err != nil {
return Decision{}, fmt.Errorf("decision %s: %w", row.ID, err)
}
return kept, nil
}