Files
archipelago/sdk/judgegate/record.go
T

108 lines
3.5 KiB
Go

package judgegate
import (
"crypto/sha256"
"encoding/hex"
"time"
"github.com/samcharles93/archipelago/sdk/guard"
"github.com/samcharles93/archipelago/sdk/systemone"
)
// Standing is a judge's standing from the recorded eval: whether its verdicts
// count, and if not, why.
type Standing struct {
Trusted bool
Reason string
EvalDate string
}
// StandingOf reads a shipped evaluation result and says where it leaves model at
// now. An unreadable result leaves the judge untrusted, with the cause as reason.
func StandingOf(evalJSON []byte, model string, now time.Time, maxAge time.Duration) Standing {
s := Standing{Reason: AdvisoryReason(evalJSON, model, now, maxAge)}
s.Trusted = s.Reason == ""
if rec, err := ParseRecord(evalJSON); err == nil {
s.EvalDate = rec.Date
}
return s
}
// Call is one judge call: who answered, with what result id, and what it was
// judged on beyond the action.
type Call struct {
Model string
ResultID string
// Answers is the service's answers as JSON.
Answers []byte
// Basis is the text the judge was given beyond the action.
Basis string
// Request and Response are the exact bodies sent and received, and the
// path, status and latency that describe the call. They are set even when
// the call failed.
Request []byte
Response []byte
EndpointPath string
HTTPStatus int
LatencyMS int64
}
// CallOf describes a call made to model that returned res, whether or not it
// succeeded: a failed call still names the model, the basis it was given, and
// what was sent and received.
func CallOf(model, basis string, res systemone.Result) Call {
c := Call{
Model: model, ResultID: res.ID, Basis: basis,
Request: res.Request, Response: res.Response,
EndpointPath: res.EndpointPath, HTTPStatus: res.HTTPStatus, LatencyMS: res.Latency.Milliseconds(),
}
if res.Model != "" {
c.Model = res.Model
}
if len(res.Answers) > 0 {
c.Answers = res.AnswersJSON()
}
return c
}
// Attach copies what the call sent and received into a review record, so a
// case's answer keeps its payload even when the judge failed.
func (c Call) Attach(g Graded) Graded {
g.Request, g.Response = string(c.Request), string(c.Response)
g.EndpointPath, g.HTTPStatus, g.LatencyMS = c.EndpointPath, c.HTTPStatus, c.LatencyMS
return g
}
// Record is what the call answered, kept whole so the verdict can be audited
// without asking the model again.
func (c Call) Record(answer string, confidence float64, reason string, st Standing) guard.JudgeRecord {
r := guard.JudgeRecord{
Model: c.Model, ResultID: c.ResultID, Answer: answer, Confidence: confidence, Reason: reason,
Basis: c.Basis, Answers: c.Answers,
Trusted: st.Trusted, AdvisoryReason: st.Reason, EvalDate: st.EvalDate,
Request: c.Request, Response: c.Response,
EndpointPath: c.EndpointPath, HTTPStatus: c.HTTPStatus, LatencyMS: c.LatencyMS,
}
if c.Basis != "" {
sum := sha256.Sum256([]byte(c.Basis))
r.BasisDigest = hex.EncodeToString(sum[:])
}
return r
}
// Failed is the record of a call that errored: the error is the reason.
func (c Call) Failed(err error, st Standing) guard.JudgeRecord {
return c.Record("", 0, "the judge call failed: "+err.Error(), st)
}
// FailedVerdict is what a guard answers when its judge could not: the action
// holds at propose and wait, as an error would hold it, and the record keeps
// what was attempted.
func FailedVerdict(rec guard.JudgeRecord) guard.Verdict {
return guard.Verdict{
Ceiling: guard.ProposeAndWait,
BlindSpots: []string{rec.Reason},
Judges: []guard.JudgeRecord{rec},
}
}