Files
archipelago/guards/policy/internal/judge/judge.go
T

151 lines
4.3 KiB
Go

// Package judge asks a System One model whether an action breaks the policy.
package judge
import (
"context"
"embed"
"fmt"
"strings"
"github.com/samcharles93/archipelago/sdk/guard"
"github.com/samcharles93/archipelago/sdk/judgegate"
"github.com/samcharles93/archipelago/sdk/systemone"
)
//go:embed prompts/*.json
var promptFiles embed.FS
const (
// DefaultMinConfidence is the confidence below which a violation is
// reported as unclear.
DefaultMinConfidence = 0.7
noRule = "none"
// maxRules bounds the question the judge is asked.
maxRules = 40
)
// The labels the verdict question answers with.
const (
Complies = "complies"
Violates = "violates"
Unclear = "unclear"
)
// Judge asks about actions against one set of rules.
type Judge struct {
client judgegate.Client
rules []string
minConfidence float64
questions map[string]systemone.Question
}
// ParseRules reads one rule per non-empty line, without list bullets.
func ParseRules(text string) []string {
var rules []string
for line := range strings.Lines(text) {
line = strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(line), "-*"))
if line != "" {
rules = append(rules, line)
}
}
return rules
}
// New returns a judge of rules.
func New(client judgegate.Client, rules []string, minConfidence float64) (*Judge, error) {
if len(rules) > maxRules {
return nil, fmt.Errorf("policy has %d rules, at most %d can be judged", len(rules), maxRules)
}
verdict, err := judgegate.LoadChoice(promptFiles, "verdict")
if err != nil {
return nil, err
}
rule, err := judgegate.LoadChoice(promptFiles, "rule")
if err != nil {
return nil, err
}
criteria := map[string]string{noRule: rule.Criteria[noRule]}
for i, r := range rules {
criteria[ruleKey(i)] = r
}
return &Judge{
client: client, rules: rules, minConfidence: minConfidence,
questions: map[string]systemone.Question{
"verdict": verdict.Question(),
"rule": systemone.Choice(rule.Instructions, criteria),
},
}, nil
}
func ruleKey(i int) string { return fmt.Sprintf("rule %d", i+1) }
// Outcome is what the judge found about one action.
type Outcome struct {
Label string
Confidence float64
// Rule is the policy rule the answer rests on, empty for none.
Rule string
// Call is what the judge was asked and returned, set even when it failed.
Call judgegate.Call
}
// Assess asks the judge whether the action breaks the policy. A violation the
// judge is not confident of is reported as unclear.
func (j *Judge) Assess(ctx context.Context, a judgegate.Action) (Outcome, error) {
state := map[string]any{"policy": j.rules, "action": a}
res, err := j.client.Classify(ctx, state, j.questions)
call := judgegate.CallOf(j.client.ModelID(), strings.Join(j.rules, "\n"), res)
if err != nil {
return Outcome{Call: call}, fmt.Errorf("judge: %w", err)
}
v := res.Answers["verdict"]
out := Outcome{Label: v.Label(), Confidence: v.Confidence, Call: call}
if out.Label == Violates && out.Confidence > 0 && out.Confidence < j.minConfidence {
out.Label = Unclear
}
picked := res.Answers["rule"].Label()
for i, r := range j.rules {
if picked == ruleKey(i) {
out.Rule = r
}
}
return out, nil
}
// Verdict turns what the judge found into a level and its reason. A complying
// action acts quietly, an unclear one is reported and goes ahead, and a
// violation waits for a person.
func (o Outcome) Verdict(st judgegate.Standing) guard.Verdict {
v := o.verdict()
v.Judges = []guard.JudgeRecord{o.Call.Record(o.Label, o.Confidence, o.evidence(), st)}
return v
}
func (o Outcome) verdict() guard.Verdict {
switch o.Label {
case Violates:
return guard.Verdict{Ceiling: guard.ProposeAndWait, Findings: []guard.Finding{{
Text: "the action breaks the organisation's policy", Evidence: o.evidence(), Strength: strength(o.Confidence),
}}}
case Unclear:
return guard.Verdict{Ceiling: guard.ActAndTell, Findings: []guard.Finding{{
Text: "the policy may cover this action but its description does not settle it", Evidence: o.evidence(), Strength: guard.Weak,
}}}
}
return guard.Verdict{Ceiling: guard.ActQuietly}
}
func (o Outcome) evidence() string {
if o.Rule == "" {
return "no single rule"
}
return "rule: " + o.Rule
}
func strength(confidence float64) guard.Strength {
if confidence >= 0.9 {
return guard.Strong
}
return guard.Moderate
}