feat(guard): a judge record keeps the request, response, path, status and latency of every model call

This commit is contained in:
sam committed 2026-11-16 12:06:05 +11:00
1 parent e909aee155
commit 124d2033fc
7 files changed
+120 -4

No files matched your search

+56 -3
View File
@@ -905,7 +905,18 @@ type JudgeRecord struct {
Trusted bool `protobuf:"varint,9,opt,name=trusted,proto3" json:"trusted,omitempty"`
AdvisoryReason string `protobuf:"bytes,10,opt,name=advisory_reason,json=advisoryReason,proto3" json:"advisory_reason,omitempty"`
// eval_date is the date of the eval the judge's standing comes from.
EvalDate string `protobuf:"bytes,11,opt,name=eval_date,json=evalDate,proto3" json:"eval_date,omitempty"`
EvalDate string `protobuf:"bytes,11,opt,name=eval_date,json=evalDate,proto3" json:"eval_date,omitempty"`
// request is the exact body sent to the judge endpoint, and response the
// exact body it returned, both kept for audit. A failed call records what
// was sent and whatever came back.
Request []byte `protobuf:"bytes,12,opt,name=request,proto3" json:"request,omitempty"`
Response []byte `protobuf:"bytes,13,opt,name=response,proto3" json:"response,omitempty"`
// endpoint_path is the path the request was sent to, without the host.
EndpointPath string `protobuf:"bytes,14,opt,name=endpoint_path,json=endpointPath,proto3" json:"endpoint_path,omitempty"`
// http_status is the response status, or 0 when no response arrived.
HttpStatus int32 `protobuf:"varint,15,opt,name=http_status,json=httpStatus,proto3" json:"http_status,omitempty"`
// latency_ms is how long the call took, measured by the caller.
LatencyMs int64 `protobuf:"varint,16,opt,name=latency_ms,json=latencyMs,proto3" json:"latency_ms,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1017,6 +1028,41 @@ func (x *JudgeRecord) GetEvalDate() string {
return ""
}
func (x *JudgeRecord) GetRequest() []byte {
if x != nil {
return x.Request
}
return nil
}
func (x *JudgeRecord) GetResponse() []byte {
if x != nil {
return x.Response
}
return nil
}
func (x *JudgeRecord) GetEndpointPath() string {
if x != nil {
return x.EndpointPath
}
return ""
}
func (x *JudgeRecord) GetHttpStatus() int32 {
if x != nil {
return x.HttpStatus
}
return 0
}
func (x *JudgeRecord) GetLatencyMs() int64 {
if x != nil {
return x.LatencyMs
}
return 0
}
var File_guard_v1_guard_proto protoreflect.FileDescriptor
const file_guard_v1_guard_proto_rawDesc = "" +
@@ -1078,7 +1124,7 @@ const file_guard_v1_guard_proto_rawDesc = "" +
"\bfindings\x18\x02 \x03(\v2\x11.guard.v1.FindingR\bfindings\x12\x1f\n" +
"\vblind_spots\x18\x03 \x03(\tR\n" +
"blindSpots\x12-\n" +
"\x06judges\x18\x04 \x03(\v2\x15.guard.v1.JudgeRecordR\x06judges\"\xc3\x02\n" +
"\x06judges\x18\x04 \x03(\v2\x15.guard.v1.JudgeRecordR\x06judges\"\xde\x03\n" +
"\vJudgeRecord\x12\x14\n" +
"\x05model\x18\x01 \x01(\tR\x05model\x12\x1b\n" +
"\tresult_id\x18\x02 \x01(\tR\bresultId\x12\x16\n" +
@@ -1093,7 +1139,14 @@ const file_guard_v1_guard_proto_rawDesc = "" +
"\atrusted\x18\t \x01(\bR\atrusted\x12'\n" +
"\x0fadvisory_reason\x18\n" +
" \x01(\tR\x0eadvisoryReason\x12\x1b\n" +
"\teval_date\x18\v \x01(\tR\bevalDate*\x95\x01\n" +
"\teval_date\x18\v \x01(\tR\bevalDate\x12\x18\n" +
"\arequest\x18\f \x01(\fR\arequest\x12\x1a\n" +
"\bresponse\x18\r \x01(\fR\bresponse\x12#\n" +
"\rendpoint_path\x18\x0e \x01(\tR\fendpointPath\x12\x1f\n" +
"\vhttp_status\x18\x0f \x01(\x05R\n" +
"httpStatus\x12\x1d\n" +
"\n" +
"latency_ms\x18\x10 \x01(\x03R\tlatencyMs*\x95\x01\n" +
"\rReversibility\x12\x1d\n" +
"\x19REVERSIBILITY_UNSPECIFIED\x10\x00\x12&\n" +
"\"REVERSIBILITY_UNDONE_AUTOMATICALLY\x10\x01\x12\x1d\n" +
+28
View File
@@ -2,6 +2,8 @@ package connectors
import (
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"time"
@@ -73,6 +75,13 @@ type GuardJudge struct {
Trusted bool
AdvisoryReason string
EvalDate string
// Request and Response are what the judge endpoint was sent and what it
// returned, both kept as the exact bodies for audit.
Request jsontext.Value
Response jsontext.Value
EndpointPath string
HTTPStatus int
LatencyMS int64
}
// Guard is a running guard extension.
@@ -129,11 +138,30 @@ func (g *Guard) Judge(ctx context.Context, r GuardRequest) (GuardAnswer, error)
Model: j.GetModel(), ResultID: j.GetResultId(), Answer: j.GetAnswer(), Confidence: j.GetConfidence(),
Reason: j.GetReason(), Basis: j.GetBasis(), BasisDigest: j.GetBasisDigest(), Answers: j.GetAnswers(),
Trusted: j.GetTrusted(), AdvisoryReason: j.GetAdvisoryReason(), EvalDate: j.GetEvalDate(),
Request: jsonValue(j.GetRequest()), Response: jsonValue(j.GetResponse()),
EndpointPath: j.GetEndpointPath(), HTTPStatus: int(j.GetHttpStatus()), LatencyMS: j.GetLatencyMs(),
})
}
return a, nil
}
// jsonValue keeps a recorded body verbatim when it is JSON, and quotes it when
// it is not, so the stored verdict always carries a valid JSON value.
func jsonValue(b []byte) jsontext.Value {
if len(b) == 0 {
return nil
}
v := jsontext.Value(b)
if v.IsValid() {
return v
}
quoted, err := json.Marshal(string(b))
if err != nil {
return nil
}
return quoted
}
func strengthName(s gpb.Strength) string {
switch s {
case gpb.Strength_STRENGTH_WEAK:
+2
View File
@@ -91,6 +91,8 @@ func (e extensionGuard) Judge(ctx context.Context, a Action) (Verdict, error) {
Model: j.Model, ResultID: j.ResultID, Answer: j.Answer, Confidence: j.Confidence, Reason: j.Reason,
Basis: j.Basis, BasisDigest: j.BasisDigest, Answers: j.Answers,
Trusted: j.Trusted, AdvisoryReason: j.AdvisoryReason, EvalDate: j.EvalDate,
Request: j.Request, Response: j.Response,
EndpointPath: j.EndpointPath, HTTPStatus: j.HTTPStatus, LatencyMS: j.LatencyMS,
})
}
return v, nil
+7
View File
@@ -91,6 +91,13 @@ type JudgeRecord struct {
Trusted bool `json:"trusted,omitzero"`
AdvisoryReason string `json:"advisory_reason,omitzero"`
EvalDate string `json:"eval_date,omitzero"`
// Request and Response are the exact bodies the judge endpoint was sent
// and returned, kept for audit.
Request jsontext.Value `json:"request,omitzero"`
Response jsontext.Value `json:"response,omitzero"`
EndpointPath string `json:"endpoint_path,omitzero"`
HTTPStatus int `json:"http_status,omitzero"`
LatencyMS int64 `json:"latency_ms,omitzero"`
}
// Verdict is one guard's judgement of one action.
+11
View File
@@ -152,4 +152,15 @@ message JudgeRecord {
string advisory_reason = 10;
// eval_date is the date of the eval the judge's standing comes from.
string eval_date = 11;
// request is the exact body sent to the judge endpoint, and response the
// exact body it returned, both kept for audit. A failed call records what
// was sent and whatever came back.
bytes request = 12;
bytes response = 13;
// endpoint_path is the path the request was sent to, without the host.
string endpoint_path = 14;
// http_status is the response status, or 0 when no response arrived.
int32 http_status = 15;
// latency_ms is how long the call took, measured by the caller.
int64 latency_ms = 16;
}
+11 -1
View File
@@ -34,6 +34,7 @@ const approvalRows = computed(() => {
});
const json = (v: unknown) => JSON.stringify(v, null, 2);
const body = (v: unknown) => (typeof v === "string" ? v : json(v));
const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary");
</script>
@@ -76,9 +77,18 @@ const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary");
<span v-if="j.eval_date" class="text-caption text-muted-foreground">Eval {{ j.eval_date }}</span>
<span v-if="j.basis_digest" class="text-caption text-muted-foreground">Basis {{ j.basis_digest.slice(0, 12) }}</span>
<span v-if="j.result_id" class="text-caption text-muted-foreground">Result {{ j.result_id }}</span>
<details v-if="j.basis || j.answers" class="text-caption text-muted-foreground">
<details v-if="j.basis || j.answers || j.request || j.response" class="text-caption text-muted-foreground">
<summary>Judge record</summary>
<span v-if="j.endpoint_path" class="block">
{{ j.endpoint_path }}<template v-if="j.http_status"> · HTTP {{ j.http_status }}</template><template v-if="j.latency_ms !== undefined"> · {{ j.latency_ms }} ms</template>
</span>
<span v-if="j.basis" class="block">Basis</span>
<pre v-if="j.basis" class="whitespace-pre-wrap">{{ j.basis }}</pre>
<span v-if="j.request" class="block">Request body</span>
<pre v-if="j.request" class="whitespace-pre-wrap">{{ body(j.request) }}</pre>
<span v-if="j.response" class="block">Response body</span>
<pre v-if="j.response" class="whitespace-pre-wrap">{{ body(j.response) }}</pre>
<span v-if="j.answers" class="block">Answers</span>
<pre v-if="j.answers" class="whitespace-pre-wrap">{{ json(j.answers) }}</pre>
</details>
</div>
+5
View File
@@ -91,6 +91,11 @@ export interface JudgeRecord {
trusted?: boolean;
advisory_reason?: string;
eval_date?: string;
request?: unknown;
response?: unknown;
endpoint_path?: string;
http_status?: number;
latency_ms?: number;
}
export interface Verdict {