49 lines
2.4 KiB
Go
49 lines
2.4 KiB
Go
package main
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archipelago/guards/integrity/internal/judge"
|
|
"github.com/samcharles93/archipelago/sdk/guard"
|
|
)
|
|
|
|
// TestJudgeCountsOnlyWhileItsRecordedEvalPasses guards the rule that makes the
|
|
// judge safe to ship: it sets a ceiling only for the model the eval measured,
|
|
// while the eval is fresh and clears every threshold.
|
|
func TestJudgeCountsOnlyWhileItsRecordedEvalPasses(t *testing.T) {
|
|
const passing = `{"model":"m","date":"2026-11-01","cases":34,"judged":{"recall":0.9,"precision":0.9,"false_flag_rate":0.02},"threshold":{"min_recall":0.8,"min_precision":0.8,"max_false_flag_rate":0.05}}`
|
|
const lowRecall = `{"model":"m","date":"2026-11-01","cases":34,"judged":{"recall":0.7,"precision":0.9,"false_flag_rate":0.02},"threshold":{"min_recall":0.8,"min_precision":0.8,"max_false_flag_rate":0.05}}`
|
|
const erred = `{"model":"m","date":"2026-11-01","cases":34,"judge_errors":1,"judged":{"recall":0.9,"precision":0.9,"false_flag_rate":0.02},"threshold":{"min_recall":0.8,"min_precision":0.8,"max_false_flag_rate":0.05}}`
|
|
now := func() time.Time { return time.Date(2026, 11, 16, 0, 0, 0, 0, time.UTC) }
|
|
tests := []struct {
|
|
name string
|
|
eval string
|
|
model string
|
|
settings map[string]string
|
|
want judge.Authority
|
|
}{
|
|
{name: "a passing fresh eval for the model counts", eval: passing, model: "m", want: judge.AuthoritySignals},
|
|
{name: "a missed threshold stays advisory", eval: lowRecall, model: "m", want: judge.AuthorityAdvisory},
|
|
{name: "a failed judge call stays advisory", eval: erred, model: "m", want: judge.AuthorityAdvisory},
|
|
{name: "another model stays advisory", eval: passing, model: "other", want: judge.AuthorityAdvisory},
|
|
{name: "a stale eval stays advisory", eval: passing, model: "m", settings: map[string]string{"eval_max_age_days": "10"}, want: judge.AuthorityAdvisory},
|
|
{name: "an unreadable eval stays advisory", eval: `{}`, model: "m", want: judge.AuthorityAdvisory},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
settings := map[string]string{"judge_base_url": "http://judge.invalid/v1", "judge_model": tt.model}
|
|
for k, v := range tt.settings {
|
|
settings[k] = v
|
|
}
|
|
g, err := newIntegrity(guard.Config{Settings: settings}, []byte(tt.eval), now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if g.opts.Authority != tt.want {
|
|
t.Fatalf("authority = %q, want %q (%s)", g.opts.Authority, tt.want, g.advised)
|
|
}
|
|
})
|
|
}
|
|
}
|