Files
archipelago/sdk/judgegate/judgegate.go
T

312 lines
10 KiB
Go

// Package judgegate is what a guard that asks a System One judge shares: the
// settings that name the judge, the recorded evaluation that decides whether
// its verdicts count, and the measurement that produces that record.
//
// A guard's judge is advisory, its verdicts recorded and no ceiling set, until
// the evaluation shipped with the guard shows the configured model passing its
// threshold. The record is read once at start. It earns trust only for the
// model it measured, only while it is fresh, and only with no failed judge call.
package judgegate
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/samcharles93/archipelago/sdk/decisions"
"github.com/samcharles93/archipelago/sdk/guard"
"github.com/samcharles93/archipelago/sdk/systemone"
)
// DefaultMaxAgeDays is how long a passing evaluation stays trusted.
const DefaultMaxAgeDays = 90
// Client is a judge endpoint a guard asks. System One and the OpenAI
// Decisions API answer the same typed questions through it.
type Client interface {
// ModelID is the model this client asks.
ModelID() string
// Deployment names the endpoint in reports, such as "judge" or
// "openai-decisions".
Deployment() string
Classify(ctx context.Context, state any, questions map[string]systemone.Question) (systemone.Result, error)
}
// The judge backends judge_api selects.
const (
APISystemOne = "systemone"
APIOpenAIDecisions = "openai-decisions"
)
// NewClient builds the judge the settings name. judge_api selects systemone
// (the default) or openai-decisions. System One needs judge_base_url and
// judge_model; the Decisions API defaults its base URL and model. ok is false
// when a required setting is empty.
func NewClient(cfg guard.Config) (Client, bool, error) {
switch api := strings.TrimSpace(cfg.Settings["judge_api"]); api {
case "", APISystemOne:
base, model := cfg.Settings["judge_base_url"], cfg.Settings["judge_model"]
if base == "" || model == "" {
return nil, false, nil
}
return systemone.Client{BaseURL: base, Model: model, APIKey: cfg.Secrets["judge_api_key"], Label: "judge"}, true, nil
case APIOpenAIDecisions:
base := cfg.Settings["judge_base_url"]
if base == "" {
base = decisions.DefaultBaseURL
}
model := cfg.Settings["judge_model"]
if model == "" {
model = decisions.DefaultModel
}
key := cfg.Secrets["judge_api_key"]
if key == "" {
key = os.Getenv("OPENAI_API_KEY")
}
return decisions.Client{BaseURL: base, Model: model, APIKey: key, Label: APIOpenAIDecisions}, true, nil
default:
return nil, false, fmt.Errorf("judge_api %q must be %s or %s", api, APISystemOne, APIOpenAIDecisions)
}
}
// EvalClient builds the judge named on an eval command line.
func EvalClient(api, base, model, key string) (Client, error) {
switch api {
case "", APISystemOne:
if base == "" || model == "" {
return nil, errors.New("-base and -model are required for systemone")
}
return systemone.Client{BaseURL: base, Model: model, APIKey: key, Label: "judge"}, nil
case APIOpenAIDecisions:
if base == "" {
base = decisions.DefaultBaseURL
}
if model == "" {
model = decisions.DefaultModel
}
return decisions.Client{BaseURL: base, Model: model, APIKey: key, Label: APIOpenAIDecisions}, nil
default:
return nil, fmt.Errorf("-api %q must be %s or %s", api, APISystemOne, APIOpenAIDecisions)
}
}
// MaxAge is how long a passing evaluation stays trusted, from eval_max_age_days.
func MaxAge(cfg guard.Config) (time.Duration, error) {
days, err := PositiveNumber(cfg.Settings["eval_max_age_days"], DefaultMaxAgeDays)
if err != nil {
return 0, fmt.Errorf("eval_max_age_days: %w", err)
}
return time.Duration(days * 24 * float64(time.Hour)), nil
}
// PositiveNumber parses a setting that must be above zero, or returns fallback
// when the setting is empty.
func PositiveNumber(s string, fallback float64) (float64, error) {
if s == "" {
return fallback, nil
}
n, err := strconv.ParseFloat(s, 64)
if err != nil || n <= 0 {
return 0, fmt.Errorf("%q is not a positive number", s)
}
return n, nil
}
// Threshold is the floor a judge must clear to be trusted with a ceiling. The
// positive class is whatever the guard exists to flag.
type Threshold struct {
MinRecall float64 `json:"min_recall"`
MinPrecision float64 `json:"min_precision"`
MaxFalseFlagRate float64 `json:"max_false_flag_rate"`
// Why says in one line where the numbers come from.
Why string `json:"why"`
}
// Score is a judge's measured quality on the positive class.
type Score struct {
Accuracy float64 `json:"accuracy"`
Recall float64 `json:"recall"`
Precision float64 `json:"precision"`
FalseFlagRate float64 `json:"false_flag_rate"`
}
// Record is the recorded outcome of measuring a judge model against labelled
// cases.
type Record struct {
Model string `json:"model"`
// Date is the day the measurement was taken, as YYYY-MM-DD.
Date string `json:"date"`
Cases int `json:"cases"`
// JudgeErrors counts judge calls that failed during the run.
JudgeErrors int `json:"judge_errors"`
Judged Score `json:"judged"`
Threshold Threshold `json:"threshold"`
}
// ParseRecord decodes a recorded result.
func ParseRecord(data []byte) (Record, error) {
var r Record
if err := json.Unmarshal(data, &r); err != nil {
return Record{}, fmt.Errorf("decode eval result: %w", err)
}
return r, r.Valid()
}
// Valid reports whether the record names a model, a date and the cases it ran.
func (r Record) Valid() error {
if _, err := time.Parse(time.DateOnly, r.Date); err != nil {
return fmt.Errorf("eval result date: %w", err)
}
if r.Model == "" || r.Cases == 0 {
return errors.New("eval result names no model or no cases")
}
return nil
}
// Shortfalls lists every way the judged scores miss the threshold, or a failed
// judge call. Empty means the judge passed.
func (r Record) Shortfalls() []string {
var out []string
if r.JudgeErrors > 0 {
out = append(out, fmt.Sprintf("%d judge calls failed", r.JudgeErrors))
}
if r.Judged.Recall < r.Threshold.MinRecall {
out = append(out, fmt.Sprintf("recall %.2f is below %.2f", r.Judged.Recall, r.Threshold.MinRecall))
}
if r.Judged.Precision < r.Threshold.MinPrecision {
out = append(out, fmt.Sprintf("precision %.2f is below %.2f", r.Judged.Precision, r.Threshold.MinPrecision))
}
if r.Judged.FalseFlagRate > r.Threshold.MaxFalseFlagRate {
out = append(out, fmt.Sprintf("false flag rate %.2f is above %.2f", r.Judged.FalseFlagRate, r.Threshold.MaxFalseFlagRate))
}
return out
}
// AdvisoryReason says why the judge named by model must stay advisory at now, or
// is empty when it may count. A result earns trust only for the model it
// measured, and only for maxAge after the measurement.
func (r Record) AdvisoryReason(model string, now time.Time, maxAge time.Duration) string {
if r.Model != model {
return fmt.Sprintf("the recorded eval measured %q, not %q", r.Model, model)
}
taken, err := time.Parse(time.DateOnly, r.Date)
if err != nil {
return "the recorded eval has no valid date"
}
if now.Sub(taken) > maxAge {
return "the recorded eval from " + r.Date + " is older than the allowed age"
}
if short := r.Shortfalls(); len(short) > 0 {
return "the recorded eval did not pass: " + strings.Join(short, "; ")
}
return ""
}
// AdvisoryReason parses a shipped evaluation result and says why model's
// verdicts must stay advisory, or is empty when they may count. An unreadable
// result is a reason.
func AdvisoryReason(evalJSON []byte, model string, now time.Time, maxAge time.Duration) string {
rec, err := ParseRecord(evalJSON)
if err != nil {
return err.Error()
}
return rec.AdvisoryReason(model, now, maxAge)
}
// Measured is one run over labelled cases.
type Measured struct {
Score Score
// Cases is how many cases the judge answered.
Cases int
// Errors counts cases where the judge failed; they are left out of Score.
Errors int
// Disagreed lists the index of each case where the judge and the label
// differ.
Disagreed []int
// FirstError is the first judge failure, to say what went wrong.
FirstError string
// Answers is the judge's answer to every case, in case order, for a person
// to review against the labels.
Answers []Graded
}
// Graded is the judge's answer to one labelled case.
type Graded struct {
ID string `json:"id"`
// Label is what the case file says; Answer is what the judge said.
Label string `json:"label"`
Answer string `json:"answer,omitempty"`
// Flagged is whether the answer counts as flagging the case, and Want
// whether the label does.
Flagged bool `json:"flagged"`
Want bool `json:"want"`
Confidence float64 `json:"confidence,omitempty"`
// Reason is what the answer rests on: the rule cited or the gap found.
Reason string `json:"reason,omitempty"`
Error string `json:"error,omitempty"`
// Request and Response are the exact bodies the judge endpoint was sent
// and returned, kept for review even when the call failed.
Request string `json:"request,omitempty"`
Response string `json:"response,omitempty"`
EndpointPath string `json:"endpoint_path,omitempty"`
HTTPStatus int `json:"http_status,omitempty"`
LatencyMS int64 `json:"latency_ms,omitempty"`
}
// Measure runs every case through judge and scores the answers against the
// labels. A case the judge fails on is kept in Answers and left out of Score.
func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (Graded, error)) Measured {
var m Measured
var tp, fp, fn, tn int
for i, c := range cases {
g, err := judge(ctx, c)
if err != nil {
g.Error = err.Error()
m.Answers = append(m.Answers, g)
m.Errors++
if m.FirstError == "" {
m.FirstError = g.Error
}
continue
}
m.Answers = append(m.Answers, g)
switch {
case g.Flagged && g.Want:
tp++
case g.Flagged:
fp++
m.Disagreed = append(m.Disagreed, i)
case g.Want:
fn++
m.Disagreed = append(m.Disagreed, i)
default:
tn++
}
}
m.Cases = tp + fp + fn + tn
m.Score = Score{
Accuracy: ratio(tp+tn, m.Cases),
Recall: ratio(tp, tp+fn),
Precision: ratio(tp, tp+fp),
FalseFlagRate: ratio(fp, fp+tn),
}
return m
}
func ratio(n, d int) float64 {
if d == 0 {
return 0
}
return float64(n) / float64(d)
}
// NewRecord records a measurement of model taken at at against a threshold.
func NewRecord(model string, at time.Time, m Measured, t Threshold) Record {
return Record{Model: model, Date: at.UTC().Format(time.DateOnly), Cases: m.Cases, JudgeErrors: m.Errors, Judged: m.Score, Threshold: t}
}