77 lines
2.2 KiB
Go
77 lines
2.2 KiB
Go
// eval measures the confidence guard's judge on the labelled cases and writes
|
|
// the result the guard reads at start. Run it from guards/confidence:
|
|
//
|
|
// go run ./cmd/eval -api systemone -base <base URL> -model <model> -out eval_result.json
|
|
// go run ./cmd/eval -api openai-decisions -model gpt-6-luna -out eval_result.openai-decisions.json
|
|
//
|
|
// The API key is read from CONFIDENCE_JUDGE_API_KEY for systemone and from
|
|
// OPENAI_API_KEY for openai-decisions.
|
|
// The positive class is a diagnosis its evidence does not support.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/samcharles93/archipelago/guards/confidence/internal/judge"
|
|
"github.com/samcharles93/archipelago/sdk/judgegate"
|
|
)
|
|
|
|
var threshold = judgegate.Threshold{
|
|
MinRecall: 0.8,
|
|
MinPrecision: 0.8,
|
|
MaxFalseFlagRate: 0.05,
|
|
Why: "the gate the integrity guard uses for a judge allowed to set a ceiling: flag four in five unsupported diagnoses, be right four times in five when flagging one, and flag at most one supported diagnosis in twenty",
|
|
}
|
|
|
|
type evalCase struct {
|
|
ID string `json:"id"`
|
|
judge.Case
|
|
// Label is supported, partial or unsupported.
|
|
Label string `json:"label"`
|
|
}
|
|
|
|
func main() {
|
|
if err := judgegate.RunEval(judgegate.EvalSpec[evalCase]{
|
|
Name: "confidence", KeyEnv: "CONFIDENCE_JUDGE_API_KEY", Threshold: threshold, Load: load, Judge: judgeCase,
|
|
}); err != nil {
|
|
fmt.Fprintln(os.Stderr, "eval:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func load(path string) ([]evalCase, error) {
|
|
data, err := judgegate.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var f struct {
|
|
Cases []evalCase `json:"cases"`
|
|
}
|
|
if err := judgegate.ReadJSON(data, &f); err != nil {
|
|
return nil, fmt.Errorf("cases: %w", err)
|
|
}
|
|
if len(f.Cases) == 0 {
|
|
return nil, errors.New("no cases")
|
|
}
|
|
return f.Cases, nil
|
|
}
|
|
|
|
func judgeCase(ctx context.Context, client judgegate.Client, c evalCase) (judgegate.Graded, error) {
|
|
g := judgegate.Graded{ID: c.ID, Label: c.Label, Want: c.Label == judge.Unsupported}
|
|
j, err := judge.New(client)
|
|
if err != nil {
|
|
return g, err
|
|
}
|
|
out, err := j.Assess(ctx, c.Case)
|
|
g = out.Call.Attach(g)
|
|
if err != nil {
|
|
return g, err
|
|
}
|
|
g.Answer, g.Confidence, g.Reason = out.Label, out.Confidence, out.Gap
|
|
g.Flagged = out.Label == judge.Unsupported
|
|
return g, nil
|
|
}
|