151 lines
4.3 KiB
Go
151 lines
4.3 KiB
Go
// Package judge asks a System One model whether an action breaks the policy.
|
|
package judge
|
|
|
|
import (
|
|
"context"
|
|
"embed"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/samcharles93/archipelago/sdk/guard"
|
|
"github.com/samcharles93/archipelago/sdk/judgegate"
|
|
"github.com/samcharles93/archipelago/sdk/systemone"
|
|
)
|
|
|
|
//go:embed prompts/*.json
|
|
var promptFiles embed.FS
|
|
|
|
const (
|
|
// DefaultMinConfidence is the confidence below which a violation is
|
|
// reported as unclear.
|
|
DefaultMinConfidence = 0.7
|
|
noRule = "none"
|
|
// maxRules bounds the question the judge is asked.
|
|
maxRules = 40
|
|
)
|
|
|
|
// The labels the verdict question answers with.
|
|
const (
|
|
Complies = "complies"
|
|
Violates = "violates"
|
|
Unclear = "unclear"
|
|
)
|
|
|
|
// Judge asks about actions against one set of rules.
|
|
type Judge struct {
|
|
client judgegate.Client
|
|
rules []string
|
|
minConfidence float64
|
|
questions map[string]systemone.Question
|
|
}
|
|
|
|
// ParseRules reads one rule per non-empty line, without list bullets.
|
|
func ParseRules(text string) []string {
|
|
var rules []string
|
|
for line := range strings.Lines(text) {
|
|
line = strings.TrimSpace(strings.TrimLeft(strings.TrimSpace(line), "-*"))
|
|
if line != "" {
|
|
rules = append(rules, line)
|
|
}
|
|
}
|
|
return rules
|
|
}
|
|
|
|
// New returns a judge of rules.
|
|
func New(client judgegate.Client, rules []string, minConfidence float64) (*Judge, error) {
|
|
if len(rules) > maxRules {
|
|
return nil, fmt.Errorf("policy has %d rules, at most %d can be judged", len(rules), maxRules)
|
|
}
|
|
verdict, err := judgegate.LoadChoice(promptFiles, "verdict")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rule, err := judgegate.LoadChoice(promptFiles, "rule")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
criteria := map[string]string{noRule: rule.Criteria[noRule]}
|
|
for i, r := range rules {
|
|
criteria[ruleKey(i)] = r
|
|
}
|
|
return &Judge{
|
|
client: client, rules: rules, minConfidence: minConfidence,
|
|
questions: map[string]systemone.Question{
|
|
"verdict": verdict.Question(),
|
|
"rule": systemone.Choice(rule.Instructions, criteria),
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func ruleKey(i int) string { return fmt.Sprintf("rule %d", i+1) }
|
|
|
|
// Outcome is what the judge found about one action.
|
|
type Outcome struct {
|
|
Label string
|
|
Confidence float64
|
|
// Rule is the policy rule the answer rests on, empty for none.
|
|
Rule string
|
|
// Call is what the judge was asked and returned, set even when it failed.
|
|
Call judgegate.Call
|
|
}
|
|
|
|
// Assess asks the judge whether the action breaks the policy. A violation the
|
|
// judge is not confident of is reported as unclear.
|
|
func (j *Judge) Assess(ctx context.Context, a judgegate.Action) (Outcome, error) {
|
|
state := map[string]any{"policy": j.rules, "action": a}
|
|
res, err := j.client.Classify(ctx, state, j.questions)
|
|
call := judgegate.CallOf(j.client.ModelID(), strings.Join(j.rules, "\n"), res)
|
|
if err != nil {
|
|
return Outcome{Call: call}, fmt.Errorf("judge: %w", err)
|
|
}
|
|
v := res.Answers["verdict"]
|
|
out := Outcome{Label: v.Label(), Confidence: v.Confidence, Call: call}
|
|
if out.Label == Violates && out.Confidence > 0 && out.Confidence < j.minConfidence {
|
|
out.Label = Unclear
|
|
}
|
|
picked := res.Answers["rule"].Label()
|
|
for i, r := range j.rules {
|
|
if picked == ruleKey(i) {
|
|
out.Rule = r
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Verdict turns what the judge found into a level and its reason. A complying
|
|
// action acts quietly, an unclear one is reported and goes ahead, and a
|
|
// violation waits for a person.
|
|
func (o Outcome) Verdict(st judgegate.Standing) guard.Verdict {
|
|
v := o.verdict()
|
|
v.Judges = []guard.JudgeRecord{o.Call.Record(o.Label, o.Confidence, o.evidence(), st)}
|
|
return v
|
|
}
|
|
|
|
func (o Outcome) verdict() guard.Verdict {
|
|
switch o.Label {
|
|
case Violates:
|
|
return guard.Verdict{Ceiling: guard.ProposeAndWait, Findings: []guard.Finding{{
|
|
Text: "the action breaks the organisation's policy", Evidence: o.evidence(), Strength: strength(o.Confidence),
|
|
}}}
|
|
case Unclear:
|
|
return guard.Verdict{Ceiling: guard.ActAndTell, Findings: []guard.Finding{{
|
|
Text: "the policy may cover this action but its description does not settle it", Evidence: o.evidence(), Strength: guard.Weak,
|
|
}}}
|
|
}
|
|
return guard.Verdict{Ceiling: guard.ActQuietly}
|
|
}
|
|
|
|
func (o Outcome) evidence() string {
|
|
if o.Rule == "" {
|
|
return "no single rule"
|
|
}
|
|
return "rule: " + o.Rule
|
|
}
|
|
|
|
func strength(confidence float64) guard.Strength {
|
|
if confidence >= 0.9 {
|
|
return guard.Strong
|
|
}
|
|
return guard.Moderate
|
|
}
|