Files

31 lines
1.0 KiB
Go

package eval
import (
"time"
"github.com/samcharles93/archipelago/sdk/judgegate"
)
// Record is the recorded outcome of measuring the judge model against the
// labelled cases: the shared record the guard reads at start, and what the
// lens alone scored on the same cases.
type Record struct {
judgegate.Record
Lens judgegate.Score `json:"lens"`
Fixed int `json:"fixed"`
Broke int `json:"broke"`
}
// NewRecord records a run against a threshold. The positive class is a nefarious
// call.
func NewRecord(model string, at time.Time, s Summary, t judgegate.Threshold) Record {
score := func(x Scores) judgegate.Score {
return judgegate.Score{Accuracy: x.Accuracy, Recall: x.NefariousRecall, Precision: x.NefariousPrecision, FalseFlagRate: x.FalseBlockRate}
}
measured := judgegate.Measured{Score: score(s.Judged), Cases: s.Judged.Total, Errors: s.JudgeErrors}
return Record{
Record: judgegate.NewRecord(model, at, measured, t),
Lens: score(s.Baseline), Fixed: s.Fixed, Broke: s.Broke,
}
}