diff --git a/internal/contracts/guard/v1/guard.pb.go b/internal/contracts/guard/v1/guard.pb.go index 8500ba48f..cd88142f3 100644 --- a/internal/contracts/guard/v1/guard.pb.go +++ b/internal/contracts/guard/v1/guard.pb.go @@ -905,7 +905,18 @@ type JudgeRecord struct { Trusted bool `protobuf:"varint,9,opt,name=trusted,proto3" json:"trusted,omitempty"` AdvisoryReason string `protobuf:"bytes,10,opt,name=advisory_reason,json=advisoryReason,proto3" json:"advisory_reason,omitempty"` // eval_date is the date of the eval the judge's standing comes from. - EvalDate string `protobuf:"bytes,11,opt,name=eval_date,json=evalDate,proto3" json:"eval_date,omitempty"` + EvalDate string `protobuf:"bytes,11,opt,name=eval_date,json=evalDate,proto3" json:"eval_date,omitempty"` + // request is the exact body sent to the judge endpoint, and response the + // exact body it returned, both kept for audit. A failed call records what + // was sent and whatever came back. + Request []byte `protobuf:"bytes,12,opt,name=request,proto3" json:"request,omitempty"` + Response []byte `protobuf:"bytes,13,opt,name=response,proto3" json:"response,omitempty"` + // endpoint_path is the path the request was sent to, without the host. + EndpointPath string `protobuf:"bytes,14,opt,name=endpoint_path,json=endpointPath,proto3" json:"endpoint_path,omitempty"` + // http_status is the response status, or 0 when no response arrived. + HttpStatus int32 `protobuf:"varint,15,opt,name=http_status,json=httpStatus,proto3" json:"http_status,omitempty"` + // latency_ms is how long the call took, measured by the caller. + LatencyMs int64 `protobuf:"varint,16,opt,name=latency_ms,json=latencyMs,proto3" json:"latency_ms,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1017,6 +1028,41 @@ func (x *JudgeRecord) GetEvalDate() string { return "" } +func (x *JudgeRecord) GetRequest() []byte { + if x != nil { + return x.Request + } + return nil +} + +func (x *JudgeRecord) GetResponse() []byte { + if x != nil { + return x.Response + } + return nil +} + +func (x *JudgeRecord) GetEndpointPath() string { + if x != nil { + return x.EndpointPath + } + return "" +} + +func (x *JudgeRecord) GetHttpStatus() int32 { + if x != nil { + return x.HttpStatus + } + return 0 +} + +func (x *JudgeRecord) GetLatencyMs() int64 { + if x != nil { + return x.LatencyMs + } + return 0 +} + var File_guard_v1_guard_proto protoreflect.FileDescriptor const file_guard_v1_guard_proto_rawDesc = "" + @@ -1078,7 +1124,7 @@ const file_guard_v1_guard_proto_rawDesc = "" + "\bfindings\x18\x02 \x03(\v2\x11.guard.v1.FindingR\bfindings\x12\x1f\n" + "\vblind_spots\x18\x03 \x03(\tR\n" + "blindSpots\x12-\n" + - "\x06judges\x18\x04 \x03(\v2\x15.guard.v1.JudgeRecordR\x06judges\"\xc3\x02\n" + + "\x06judges\x18\x04 \x03(\v2\x15.guard.v1.JudgeRecordR\x06judges\"\xde\x03\n" + "\vJudgeRecord\x12\x14\n" + "\x05model\x18\x01 \x01(\tR\x05model\x12\x1b\n" + "\tresult_id\x18\x02 \x01(\tR\bresultId\x12\x16\n" + @@ -1093,7 +1139,14 @@ const file_guard_v1_guard_proto_rawDesc = "" + "\atrusted\x18\t \x01(\bR\atrusted\x12'\n" + "\x0fadvisory_reason\x18\n" + " \x01(\tR\x0eadvisoryReason\x12\x1b\n" + - "\teval_date\x18\v \x01(\tR\bevalDate*\x95\x01\n" + + "\teval_date\x18\v \x01(\tR\bevalDate\x12\x18\n" + + "\arequest\x18\f \x01(\fR\arequest\x12\x1a\n" + + "\bresponse\x18\r \x01(\fR\bresponse\x12#\n" + + "\rendpoint_path\x18\x0e \x01(\tR\fendpointPath\x12\x1f\n" + + "\vhttp_status\x18\x0f \x01(\x05R\n" + + "httpStatus\x12\x1d\n" + + "\n" + + "latency_ms\x18\x10 \x01(\x03R\tlatencyMs*\x95\x01\n" + "\rReversibility\x12\x1d\n" + "\x19REVERSIBILITY_UNSPECIFIED\x10\x00\x12&\n" + "\"REVERSIBILITY_UNDONE_AUTOMATICALLY\x10\x01\x12\x1d\n" + diff --git a/modules/connectors/guards.go b/modules/connectors/guards.go index a11f694e8..d5db3f42e 100644 --- a/modules/connectors/guards.go +++ b/modules/connectors/guards.go @@ -2,6 +2,8 @@ package connectors import ( "context" + "encoding/json/jsontext" + json "encoding/json/v2" "errors" "fmt" "time" @@ -73,6 +75,13 @@ type GuardJudge struct { Trusted bool AdvisoryReason string EvalDate string + // Request and Response are what the judge endpoint was sent and what it + // returned, both kept as the exact bodies for audit. + Request jsontext.Value + Response jsontext.Value + EndpointPath string + HTTPStatus int + LatencyMS int64 } // Guard is a running guard extension. @@ -129,11 +138,30 @@ func (g *Guard) Judge(ctx context.Context, r GuardRequest) (GuardAnswer, error) Model: j.GetModel(), ResultID: j.GetResultId(), Answer: j.GetAnswer(), Confidence: j.GetConfidence(), Reason: j.GetReason(), Basis: j.GetBasis(), BasisDigest: j.GetBasisDigest(), Answers: j.GetAnswers(), Trusted: j.GetTrusted(), AdvisoryReason: j.GetAdvisoryReason(), EvalDate: j.GetEvalDate(), + Request: jsonValue(j.GetRequest()), Response: jsonValue(j.GetResponse()), + EndpointPath: j.GetEndpointPath(), HTTPStatus: int(j.GetHttpStatus()), LatencyMS: j.GetLatencyMs(), }) } return a, nil } +// jsonValue keeps a recorded body verbatim when it is JSON, and quotes it when +// it is not, so the stored verdict always carries a valid JSON value. +func jsonValue(b []byte) jsontext.Value { + if len(b) == 0 { + return nil + } + v := jsontext.Value(b) + if v.IsValid() { + return v + } + quoted, err := json.Marshal(string(b)) + if err != nil { + return nil + } + return quoted +} + func strengthName(s gpb.Strength) string { switch s { case gpb.Strength_STRENGTH_WEAK: diff --git a/modules/judgement/extensions.go b/modules/judgement/extensions.go index 042e4a345..783f6b08b 100644 --- a/modules/judgement/extensions.go +++ b/modules/judgement/extensions.go @@ -91,6 +91,8 @@ func (e extensionGuard) Judge(ctx context.Context, a Action) (Verdict, error) { Model: j.Model, ResultID: j.ResultID, Answer: j.Answer, Confidence: j.Confidence, Reason: j.Reason, Basis: j.Basis, BasisDigest: j.BasisDigest, Answers: j.Answers, Trusted: j.Trusted, AdvisoryReason: j.AdvisoryReason, EvalDate: j.EvalDate, + Request: j.Request, Response: j.Response, + EndpointPath: j.EndpointPath, HTTPStatus: j.HTTPStatus, LatencyMS: j.LatencyMS, }) } return v, nil diff --git a/modules/judgement/judgement.go b/modules/judgement/judgement.go index 3430301f6..a69c067da 100644 --- a/modules/judgement/judgement.go +++ b/modules/judgement/judgement.go @@ -91,6 +91,13 @@ type JudgeRecord struct { Trusted bool `json:"trusted,omitzero"` AdvisoryReason string `json:"advisory_reason,omitzero"` EvalDate string `json:"eval_date,omitzero"` + // Request and Response are the exact bodies the judge endpoint was sent + // and returned, kept for audit. + Request jsontext.Value `json:"request,omitzero"` + Response jsontext.Value `json:"response,omitzero"` + EndpointPath string `json:"endpoint_path,omitzero"` + HTTPStatus int `json:"http_status,omitzero"` + LatencyMS int64 `json:"latency_ms,omitzero"` } // Verdict is one guard's judgement of one action. diff --git a/proto/guard/v1/guard.proto b/proto/guard/v1/guard.proto index 285fab2e5..fda480db4 100644 --- a/proto/guard/v1/guard.proto +++ b/proto/guard/v1/guard.proto @@ -152,4 +152,15 @@ message JudgeRecord { string advisory_reason = 10; // eval_date is the date of the eval the judge's standing comes from. string eval_date = 11; + // request is the exact body sent to the judge endpoint, and response the + // exact body it returned, both kept for audit. A failed call records what + // was sent and whatever came back. + bytes request = 12; + bytes response = 13; + // endpoint_path is the path the request was sent to, without the host. + string endpoint_path = 14; + // http_status is the response status, or 0 when no response arrived. + int32 http_status = 15; + // latency_ms is how long the call took, measured by the caller. + int64 latency_ms = 16; } diff --git a/ui/src/archie/ActionTrailView.vue b/ui/src/archie/ActionTrailView.vue index 520d736be..0532954f0 100644 --- a/ui/src/archie/ActionTrailView.vue +++ b/ui/src/archie/ActionTrailView.vue @@ -34,6 +34,7 @@ const approvalRows = computed(() => { }); const json = (v: unknown) => JSON.stringify(v, null, 2); +const body = (v: unknown) => (typeof v === "string" ? v : json(v)); const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary"); @@ -76,9 +77,18 @@ const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary"); Eval {{ j.eval_date }} Basis {{ j.basis_digest.slice(0, 12) }} Result {{ j.result_id }} -
+
Judge record + + {{ j.endpoint_path }} + + Basis
{{ j.basis }}
+ Request body +
{{ body(j.request) }}
+ Response body +
{{ body(j.response) }}
+ Answers
{{ json(j.answers) }}
diff --git a/ui/src/archie/api.ts b/ui/src/archie/api.ts index 95092c165..30e76ceb6 100644 --- a/ui/src/archie/api.ts +++ b/ui/src/archie/api.ts @@ -91,6 +91,11 @@ export interface JudgeRecord { trusted?: boolean; advisory_reason?: string; eval_date?: string; + request?: unknown; + response?: unknown; + endpoint_path?: string; + http_status?: number; + latency_ms?: number; } export interface Verdict {