312 lines
10 KiB
Go
312 lines
10 KiB
Go
// Package judgegate is what a guard that asks a System One judge shares: the
|
|
// settings that name the judge, the recorded evaluation that decides whether
|
|
// its verdicts count, and the measurement that produces that record.
|
|
//
|
|
// A guard's judge is advisory, its verdicts recorded and no ceiling set, until
|
|
// the evaluation shipped with the guard shows the configured model passing its
|
|
// threshold. The record is read once at start. It earns trust only for the
|
|
// model it measured, only while it is fresh, and only with no failed judge call.
|
|
package judgegate
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archipelago/sdk/decisions"
|
|
"github.com/samcharles93/archipelago/sdk/guard"
|
|
"github.com/samcharles93/archipelago/sdk/systemone"
|
|
)
|
|
|
|
// DefaultMaxAgeDays is how long a passing evaluation stays trusted.
|
|
const DefaultMaxAgeDays = 90
|
|
|
|
// Client is a judge endpoint a guard asks. System One and the OpenAI
|
|
// Decisions API answer the same typed questions through it.
|
|
type Client interface {
|
|
// ModelID is the model this client asks.
|
|
ModelID() string
|
|
// Deployment names the endpoint in reports, such as "judge" or
|
|
// "openai-decisions".
|
|
Deployment() string
|
|
Classify(ctx context.Context, state any, questions map[string]systemone.Question) (systemone.Result, error)
|
|
}
|
|
|
|
// The judge backends judge_api selects.
|
|
const (
|
|
APISystemOne = "systemone"
|
|
APIOpenAIDecisions = "openai-decisions"
|
|
)
|
|
|
|
// NewClient builds the judge the settings name. judge_api selects systemone
|
|
// (the default) or openai-decisions. System One needs judge_base_url and
|
|
// judge_model; the Decisions API defaults its base URL and model. ok is false
|
|
// when a required setting is empty.
|
|
func NewClient(cfg guard.Config) (Client, bool, error) {
|
|
switch api := strings.TrimSpace(cfg.Settings["judge_api"]); api {
|
|
case "", APISystemOne:
|
|
base, model := cfg.Settings["judge_base_url"], cfg.Settings["judge_model"]
|
|
if base == "" || model == "" {
|
|
return nil, false, nil
|
|
}
|
|
return systemone.Client{BaseURL: base, Model: model, APIKey: cfg.Secrets["judge_api_key"], Label: "judge"}, true, nil
|
|
case APIOpenAIDecisions:
|
|
base := cfg.Settings["judge_base_url"]
|
|
if base == "" {
|
|
base = decisions.DefaultBaseURL
|
|
}
|
|
model := cfg.Settings["judge_model"]
|
|
if model == "" {
|
|
model = decisions.DefaultModel
|
|
}
|
|
key := cfg.Secrets["judge_api_key"]
|
|
if key == "" {
|
|
key = os.Getenv("OPENAI_API_KEY")
|
|
}
|
|
return decisions.Client{BaseURL: base, Model: model, APIKey: key, Label: APIOpenAIDecisions}, true, nil
|
|
default:
|
|
return nil, false, fmt.Errorf("judge_api %q must be %s or %s", api, APISystemOne, APIOpenAIDecisions)
|
|
}
|
|
}
|
|
|
|
// EvalClient builds the judge named on an eval command line.
|
|
func EvalClient(api, base, model, key string) (Client, error) {
|
|
switch api {
|
|
case "", APISystemOne:
|
|
if base == "" || model == "" {
|
|
return nil, errors.New("-base and -model are required for systemone")
|
|
}
|
|
return systemone.Client{BaseURL: base, Model: model, APIKey: key, Label: "judge"}, nil
|
|
case APIOpenAIDecisions:
|
|
if base == "" {
|
|
base = decisions.DefaultBaseURL
|
|
}
|
|
if model == "" {
|
|
model = decisions.DefaultModel
|
|
}
|
|
return decisions.Client{BaseURL: base, Model: model, APIKey: key, Label: APIOpenAIDecisions}, nil
|
|
default:
|
|
return nil, fmt.Errorf("-api %q must be %s or %s", api, APISystemOne, APIOpenAIDecisions)
|
|
}
|
|
}
|
|
|
|
// MaxAge is how long a passing evaluation stays trusted, from eval_max_age_days.
|
|
func MaxAge(cfg guard.Config) (time.Duration, error) {
|
|
days, err := PositiveNumber(cfg.Settings["eval_max_age_days"], DefaultMaxAgeDays)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("eval_max_age_days: %w", err)
|
|
}
|
|
return time.Duration(days * 24 * float64(time.Hour)), nil
|
|
}
|
|
|
|
// PositiveNumber parses a setting that must be above zero, or returns fallback
|
|
// when the setting is empty.
|
|
func PositiveNumber(s string, fallback float64) (float64, error) {
|
|
if s == "" {
|
|
return fallback, nil
|
|
}
|
|
n, err := strconv.ParseFloat(s, 64)
|
|
if err != nil || n <= 0 {
|
|
return 0, fmt.Errorf("%q is not a positive number", s)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// Threshold is the floor a judge must clear to be trusted with a ceiling. The
|
|
// positive class is whatever the guard exists to flag.
|
|
type Threshold struct {
|
|
MinRecall float64 `json:"min_recall"`
|
|
MinPrecision float64 `json:"min_precision"`
|
|
MaxFalseFlagRate float64 `json:"max_false_flag_rate"`
|
|
// Why says in one line where the numbers come from.
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
// Score is a judge's measured quality on the positive class.
|
|
type Score struct {
|
|
Accuracy float64 `json:"accuracy"`
|
|
Recall float64 `json:"recall"`
|
|
Precision float64 `json:"precision"`
|
|
FalseFlagRate float64 `json:"false_flag_rate"`
|
|
}
|
|
|
|
// Record is the recorded outcome of measuring a judge model against labelled
|
|
// cases.
|
|
type Record struct {
|
|
Model string `json:"model"`
|
|
// Date is the day the measurement was taken, as YYYY-MM-DD.
|
|
Date string `json:"date"`
|
|
Cases int `json:"cases"`
|
|
// JudgeErrors counts judge calls that failed during the run.
|
|
JudgeErrors int `json:"judge_errors"`
|
|
Judged Score `json:"judged"`
|
|
Threshold Threshold `json:"threshold"`
|
|
}
|
|
|
|
// ParseRecord decodes a recorded result.
|
|
func ParseRecord(data []byte) (Record, error) {
|
|
var r Record
|
|
if err := json.Unmarshal(data, &r); err != nil {
|
|
return Record{}, fmt.Errorf("decode eval result: %w", err)
|
|
}
|
|
return r, r.Valid()
|
|
}
|
|
|
|
// Valid reports whether the record names a model, a date and the cases it ran.
|
|
func (r Record) Valid() error {
|
|
if _, err := time.Parse(time.DateOnly, r.Date); err != nil {
|
|
return fmt.Errorf("eval result date: %w", err)
|
|
}
|
|
if r.Model == "" || r.Cases == 0 {
|
|
return errors.New("eval result names no model or no cases")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Shortfalls lists every way the judged scores miss the threshold, or a failed
|
|
// judge call. Empty means the judge passed.
|
|
func (r Record) Shortfalls() []string {
|
|
var out []string
|
|
if r.JudgeErrors > 0 {
|
|
out = append(out, fmt.Sprintf("%d judge calls failed", r.JudgeErrors))
|
|
}
|
|
if r.Judged.Recall < r.Threshold.MinRecall {
|
|
out = append(out, fmt.Sprintf("recall %.2f is below %.2f", r.Judged.Recall, r.Threshold.MinRecall))
|
|
}
|
|
if r.Judged.Precision < r.Threshold.MinPrecision {
|
|
out = append(out, fmt.Sprintf("precision %.2f is below %.2f", r.Judged.Precision, r.Threshold.MinPrecision))
|
|
}
|
|
if r.Judged.FalseFlagRate > r.Threshold.MaxFalseFlagRate {
|
|
out = append(out, fmt.Sprintf("false flag rate %.2f is above %.2f", r.Judged.FalseFlagRate, r.Threshold.MaxFalseFlagRate))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// AdvisoryReason says why the judge named by model must stay advisory at now, or
|
|
// is empty when it may count. A result earns trust only for the model it
|
|
// measured, and only for maxAge after the measurement.
|
|
func (r Record) AdvisoryReason(model string, now time.Time, maxAge time.Duration) string {
|
|
if r.Model != model {
|
|
return fmt.Sprintf("the recorded eval measured %q, not %q", r.Model, model)
|
|
}
|
|
taken, err := time.Parse(time.DateOnly, r.Date)
|
|
if err != nil {
|
|
return "the recorded eval has no valid date"
|
|
}
|
|
if now.Sub(taken) > maxAge {
|
|
return "the recorded eval from " + r.Date + " is older than the allowed age"
|
|
}
|
|
if short := r.Shortfalls(); len(short) > 0 {
|
|
return "the recorded eval did not pass: " + strings.Join(short, "; ")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// AdvisoryReason parses a shipped evaluation result and says why model's
|
|
// verdicts must stay advisory, or is empty when they may count. An unreadable
|
|
// result is a reason.
|
|
func AdvisoryReason(evalJSON []byte, model string, now time.Time, maxAge time.Duration) string {
|
|
rec, err := ParseRecord(evalJSON)
|
|
if err != nil {
|
|
return err.Error()
|
|
}
|
|
return rec.AdvisoryReason(model, now, maxAge)
|
|
}
|
|
|
|
// Measured is one run over labelled cases.
|
|
type Measured struct {
|
|
Score Score
|
|
// Cases is how many cases the judge answered.
|
|
Cases int
|
|
// Errors counts cases where the judge failed; they are left out of Score.
|
|
Errors int
|
|
// Disagreed lists the index of each case where the judge and the label
|
|
// differ.
|
|
Disagreed []int
|
|
// FirstError is the first judge failure, to say what went wrong.
|
|
FirstError string
|
|
// Answers is the judge's answer to every case, in case order, for a person
|
|
// to review against the labels.
|
|
Answers []Graded
|
|
}
|
|
|
|
// Graded is the judge's answer to one labelled case.
|
|
type Graded struct {
|
|
ID string `json:"id"`
|
|
// Label is what the case file says; Answer is what the judge said.
|
|
Label string `json:"label"`
|
|
Answer string `json:"answer,omitempty"`
|
|
// Flagged is whether the answer counts as flagging the case, and Want
|
|
// whether the label does.
|
|
Flagged bool `json:"flagged"`
|
|
Want bool `json:"want"`
|
|
Confidence float64 `json:"confidence,omitempty"`
|
|
// Reason is what the answer rests on: the rule cited or the gap found.
|
|
Reason string `json:"reason,omitempty"`
|
|
Error string `json:"error,omitempty"`
|
|
// Request and Response are the exact bodies the judge endpoint was sent
|
|
// and returned, kept for review even when the call failed.
|
|
Request string `json:"request,omitempty"`
|
|
Response string `json:"response,omitempty"`
|
|
EndpointPath string `json:"endpoint_path,omitempty"`
|
|
HTTPStatus int `json:"http_status,omitempty"`
|
|
LatencyMS int64 `json:"latency_ms,omitempty"`
|
|
}
|
|
|
|
// Measure runs every case through judge and scores the answers against the
|
|
// labels. A case the judge fails on is kept in Answers and left out of Score.
|
|
func Measure[T any](ctx context.Context, cases []T, judge func(context.Context, T) (Graded, error)) Measured {
|
|
var m Measured
|
|
var tp, fp, fn, tn int
|
|
for i, c := range cases {
|
|
g, err := judge(ctx, c)
|
|
if err != nil {
|
|
g.Error = err.Error()
|
|
m.Answers = append(m.Answers, g)
|
|
m.Errors++
|
|
if m.FirstError == "" {
|
|
m.FirstError = g.Error
|
|
}
|
|
continue
|
|
}
|
|
m.Answers = append(m.Answers, g)
|
|
switch {
|
|
case g.Flagged && g.Want:
|
|
tp++
|
|
case g.Flagged:
|
|
fp++
|
|
m.Disagreed = append(m.Disagreed, i)
|
|
case g.Want:
|
|
fn++
|
|
m.Disagreed = append(m.Disagreed, i)
|
|
default:
|
|
tn++
|
|
}
|
|
}
|
|
m.Cases = tp + fp + fn + tn
|
|
m.Score = Score{
|
|
Accuracy: ratio(tp+tn, m.Cases),
|
|
Recall: ratio(tp, tp+fn),
|
|
Precision: ratio(tp, tp+fp),
|
|
FalseFlagRate: ratio(fp, fp+tn),
|
|
}
|
|
return m
|
|
}
|
|
|
|
func ratio(n, d int) float64 {
|
|
if d == 0 {
|
|
return 0
|
|
}
|
|
return float64(n) / float64(d)
|
|
}
|
|
|
|
// NewRecord records a measurement of model taken at at against a threshold.
|
|
func NewRecord(model string, at time.Time, m Measured, t Threshold) Record {
|
|
return Record{Model: model, Date: at.UTC().Format(time.DateOnly), Cases: m.Cases, JudgeErrors: m.Errors, Judged: m.Score, Threshold: t}
|
|
}
|