269 lines
8.7 KiB
Go
269 lines
8.7 KiB
Go
// Package judge adds a classifier to the auditor.
|
|
//
|
|
// The deterministic lens proves what is present in a call and can say nothing
|
|
// about what is merely suspicious. A judge covers that gap: it is a System One
|
|
// decision model asked the questions the lens cannot answer for certain, and it
|
|
// answers with labels and probabilities rather than prose.
|
|
//
|
|
// Two properties are load-bearing:
|
|
//
|
|
// - A judge that fails changes nothing. Every error path returns a verdict with
|
|
// Error set and no influence on the report, so an unreachable classifier can
|
|
// never approve anything.
|
|
// - A judge cannot overrule proof. A signal it confirms becomes a violation; a
|
|
// violation it denies stays a violation unless the operator explicitly grants
|
|
// that authority.
|
|
package judge
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/samcharles93/archipelago/guards/integrity/internal/audit"
|
|
"github.com/samcharles93/archipelago/sdk/judgegate"
|
|
"github.com/samcharles93/archipelago/sdk/systemone"
|
|
)
|
|
|
|
// Authority is how much a judge is allowed to change about the lens's verdict.
|
|
type Authority string
|
|
|
|
const (
|
|
// AuthorityAdvisory records the judge's answers and changes nothing. Use it to
|
|
// evaluate a model before letting it influence a decision.
|
|
AuthorityAdvisory Authority = "advisory"
|
|
// AuthoritySignals is the default: the judge may confirm or dismiss signals,
|
|
// but never overrule a violation, which is text proven to be present.
|
|
AuthoritySignals Authority = "signals"
|
|
// AuthorityAll also lets the judge dismiss proven violations. Off by default,
|
|
// because it allows a classifier to unblock text that is demonstrably there.
|
|
AuthorityAll Authority = "all"
|
|
)
|
|
|
|
// DefaultMinConfidence is deliberately high: a judge must be sure before it
|
|
// promotes suspicion into a block.
|
|
const DefaultMinConfidence = 0.9
|
|
|
|
// DefaultMaxFieldChars truncates each text field sent to a judge. A classifier
|
|
// with a bounded context must be told what it cannot see rather than guessing.
|
|
const DefaultMaxFieldChars = 8000
|
|
|
|
// Options configures one judgement.
|
|
type Options struct {
|
|
Authority Authority
|
|
MinConfidence float64
|
|
MaxFieldChars int
|
|
}
|
|
|
|
func (o Options) withDefaults() Options {
|
|
if o.Authority == "" {
|
|
o.Authority = AuthoritySignals
|
|
}
|
|
if o.MinConfidence <= 0 {
|
|
o.MinConfidence = DefaultMinConfidence
|
|
}
|
|
if o.MaxFieldChars <= 0 {
|
|
o.MaxFieldChars = DefaultMaxFieldChars
|
|
}
|
|
return o
|
|
}
|
|
|
|
// Valid reports whether the options are usable.
|
|
func (o Options) Valid() error {
|
|
switch o.Authority {
|
|
case AuthorityAdvisory, AuthoritySignals, AuthorityAll:
|
|
default:
|
|
return fmt.Errorf("authority %q must be advisory, signals or all", o.Authority)
|
|
}
|
|
if o.MinConfidence < 0 || o.MinConfidence > 1 {
|
|
return fmt.Errorf("min confidence %v must be between 0 and 1", o.MinConfidence)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Choice is one label answer with the service's own probabilities.
|
|
type Choice struct {
|
|
Value string
|
|
Probabilities map[string]float64
|
|
Confidence float64
|
|
}
|
|
|
|
// Verdict is everything a judge answered about one call.
|
|
//
|
|
// There is no Reason field: a System One decision model returns labels and
|
|
// probabilities, not prose, and inventing a reason on its behalf would be
|
|
// fabricating evidence.
|
|
type Verdict struct {
|
|
Provider string
|
|
Model string
|
|
Authority Authority
|
|
MinConfidence float64
|
|
Legitimacy Choice
|
|
// Validity is only asked when the call declared a contract.
|
|
Validity Choice
|
|
Signals map[string]float64 // rule id -> probability that the finding is real
|
|
Usage *systemone.Usage
|
|
// Answers is the service's answers as JSON, kept whole for the audit.
|
|
Answers []byte
|
|
// ResponseID is the service's own id for the call, kept for tracing.
|
|
ResponseID string
|
|
// Deployment names the endpoint the judge was reached through, for example
|
|
// "openrouter", which is distinct from the service that answered.
|
|
Deployment string
|
|
LatencyMS int
|
|
Truncated bool
|
|
Error string
|
|
// RequestJSON is the exact body sent to the classifier, Response the exact
|
|
// body it returned, and Questions the set it was asked. Storing them makes
|
|
// the judgement reproducible and turns the audit log into a training set
|
|
// that matches production. RequestJSON and Response are kept on a failed
|
|
// call too.
|
|
RequestJSON []byte
|
|
Response []byte
|
|
EndpointPath string
|
|
HTTPStatus int
|
|
Questions map[string]systemone.Question
|
|
}
|
|
|
|
// Ok reports whether the judge produced usable answers.
|
|
func (v Verdict) Ok() bool { return v.Error == "" }
|
|
|
|
// Judge answers questions about one call.
|
|
type Judge interface {
|
|
// Name identifies the implementation in reports.
|
|
Name() string
|
|
// Judge never returns an error: a failure is reported in Verdict.Error so a
|
|
// caller cannot forget to handle it and treat silence as approval.
|
|
Judge(ctx context.Context, b *audit.Bundle, rep *audit.Report, opts Options) Verdict
|
|
}
|
|
|
|
// SystemOne judges with a model behind a judge backend: System One locally
|
|
// through llama.cpp's /v1/systemone or hosted, or the OpenAI Decisions API.
|
|
type SystemOne struct {
|
|
Client judgegate.Client
|
|
// Kind names the backend in reports; empty means systemone.
|
|
Kind string
|
|
}
|
|
|
|
// Name identifies the implementation in reports.
|
|
func (s SystemOne) Name() string {
|
|
if s.Kind != "" {
|
|
return s.Kind
|
|
}
|
|
return judgegate.APISystemOne
|
|
}
|
|
|
|
// Judge asks one question set about the call and its deterministic findings.
|
|
func (s SystemOne) Judge(ctx context.Context, b *audit.Bundle, rep *audit.Report, opts Options) Verdict {
|
|
opts = opts.withDefaults()
|
|
v := Verdict{
|
|
Provider: s.Client.Deployment(),
|
|
Deployment: s.Client.Deployment(),
|
|
Model: s.Client.ModelID(),
|
|
Authority: opts.Authority,
|
|
MinConfidence: opts.MinConfidence,
|
|
Signals: map[string]float64{},
|
|
}
|
|
|
|
state, truncated := State(b, rep, opts)
|
|
v.Truncated = truncated
|
|
questions := Questions(b, rep, opts)
|
|
v.Questions = questions
|
|
|
|
res, err := s.Client.Classify(ctx, state, questions)
|
|
v.RequestJSON = res.Request
|
|
v.Response = res.Response
|
|
v.EndpointPath = res.EndpointPath
|
|
v.HTTPStatus = res.HTTPStatus
|
|
v.LatencyMS = int(res.Latency.Milliseconds())
|
|
v.Usage = res.Usage
|
|
if err != nil {
|
|
// Fail closed: the deterministic verdict stands.
|
|
v.ResponseID = res.ID
|
|
v.Error = err.Error()
|
|
return v
|
|
}
|
|
|
|
if res.Provider != "" {
|
|
// The service names itself, for example "TypeSafe"; the deployment label
|
|
// names the endpoint it was reached through.
|
|
v.Provider = res.Provider
|
|
}
|
|
if res.ID != "" {
|
|
v.ResponseID = res.ID
|
|
}
|
|
v.Answers = res.AnswersJSON()
|
|
if res.Model != "" {
|
|
v.Model = res.Model
|
|
}
|
|
if answer, ok := res.Answers["legitimacy"]; ok {
|
|
v.Legitimacy = Choice{Value: answer.Label(), Probabilities: answer.Probabilities, Confidence: answer.Confidence}
|
|
}
|
|
if answer, ok := res.Answers["validity"]; ok {
|
|
v.Validity = Choice{Value: answer.Label(), Probabilities: answer.Probabilities, Confidence: answer.Confidence}
|
|
}
|
|
for id, answer := range res.Answers {
|
|
rule, isFinding := findingRule(id)
|
|
if !isFinding {
|
|
continue
|
|
}
|
|
if probability, ok := answer.BoolProbability(); ok {
|
|
v.Signals[rule] = probability
|
|
}
|
|
}
|
|
return v
|
|
}
|
|
|
|
// Questions builds the question set for a call without contacting a service, and
|
|
// State builds what the judge would be shown.
|
|
//
|
|
// They are exported so the evaluation harness and the fine-tuning exporter
|
|
// construct exactly what a live judge is asked, instead of a second approximation
|
|
// of it that could drift from production.
|
|
func Questions(b *audit.Bundle, rep *audit.Report, opts Options) map[string]systemone.Question {
|
|
opts = opts.withDefaults()
|
|
questions := map[string]systemone.Question{"legitimacy": choiceQuestion("legitimacy")}
|
|
if b.Contract != nil {
|
|
questions["validity"] = choiceQuestion("validity")
|
|
}
|
|
for _, f := range rep.Findings {
|
|
if !reconsiderable(f, opts.Authority) {
|
|
continue
|
|
}
|
|
questions[findingQuestionPrefix+f.RuleID] = findingQuestion(f)
|
|
}
|
|
return questions
|
|
}
|
|
|
|
// State assembles what the judge is allowed to see; see state.go for the rules.
|
|
func State(b *audit.Bundle, rep *audit.Report, opts Options) (map[string]any, bool) {
|
|
opts = opts.withDefaults()
|
|
state, truncated := buildState(b, rep, opts)
|
|
state["authority"] = string(opts.Authority)
|
|
return state, truncated
|
|
}
|
|
|
|
// reconsiderable reports which findings a judge is allowed to ask about, given
|
|
// the authority it was granted.
|
|
func reconsiderable(f audit.Finding, authority Authority) bool {
|
|
if f.Dismissed {
|
|
return false
|
|
}
|
|
if f.Axis == audit.AxisRecord {
|
|
return false // record integrity is a fact about the artefact, not a judgement call
|
|
}
|
|
if f.Kind == audit.KindViolation {
|
|
return authority == AuthorityAll
|
|
}
|
|
return true
|
|
}
|
|
|
|
const findingQuestionPrefix = "finding_"
|
|
|
|
func findingRule(id string) (string, bool) {
|
|
if !strings.HasPrefix(id, findingQuestionPrefix) {
|
|
return "", false
|
|
}
|
|
return strings.TrimPrefix(id, findingQuestionPrefix), true
|
|
}
|