Files

244 lines
8.7 KiB
Go

// Package audit judges recorded LLM calls against a declarative rule lens.
//
// The auditor is static and offline by design: it never calls a model and never
// touches the network, so a verdict is reproducible from the bundle alone. Every
// finding carries a locator and a verbatim excerpt from the bundle, which is what
// makes a verdict reviewable rather than persuasive.
package audit
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
)
// Bundle is one recorded LLM call: the request, the response, and the context
// needed to judge it. It is the only input the auditor reads.
type Bundle struct {
ID string `json:"id,omitempty"`
Subject Subject `json:"subject,omitempty"`
Request Request `json:"request"`
Response Response `json:"response"`
Contract *Contract `json:"contract,omitempty"`
Sources []Source `json:"sources,omitempty"`
Integrity *Integrity `json:"integrity,omitempty"`
}
// Subject records who produced the call. Absent fields are a coverage finding,
// not an error: an unattributed call can be judged but not held to account.
type Subject struct {
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
Actor string `json:"actor,omitempty"`
Timestamp string `json:"timestamp,omitempty"`
Purpose string `json:"purpose,omitempty"`
}
// Request is the input side of the call.
type Request struct {
Messages []Message `json:"messages,omitempty"`
Tools []Tool `json:"tools,omitempty"`
// ToolCalls are the calls already made in this conversation, as the harness
// replays them. An agent harness resends the whole transcript on every turn, so
// the last request of a session carries its complete tool history - which is what
// makes a trajectory auditable from a single call.
//
// Note for aggregation: because the transcript is replayed, this field is not
// additive across calls. Summing it counts the same call once per subsequent
// turn. See internal/session for the dedup rule.
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
Parameters map[string]any `json:"parameters,omitempty"`
// NewFrom is the index of the first message this call added to the
// conversation; earlier messages were already present in a previous call of
// the same session. Nil means unknown, and every message counts as new. It is
// set by the record source, not sent by the caller, so it is excluded from the
// request hash.
NewFrom *int `json:"new_from,omitempty"`
}
// Message is one request-side turn.
type Message struct {
Role string `json:"role"`
Content string `json:"content,omitempty"`
}
// Tool is a tool the caller declared as available for this call.
type Tool struct {
Name string `json:"name"`
Parameters map[string]any `json:"parameters,omitempty"`
}
// Response is the output side of the call. Content is prose, Output is the
// structured payload when the caller asked for JSON, ToolCalls are requested
// actions. Tool-only responses legitimately have empty Content.
type Response struct {
Content string `json:"content,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
Output any `json:"output,omitempty"`
Usage *Usage `json:"usage,omitempty"`
Refusal bool `json:"refusal,omitempty"`
}
// ToolCall is one action the model asked the caller to perform.
type ToolCall struct {
Name string `json:"name"`
Arguments map[string]any `json:"arguments,omitempty"`
// ArgumentsRaw is the arguments exactly as the transport delivered them, when
// it delivered a string. A model can emit invalid JSON here, and that is a
// validity failure rather than a missing value, so the raw text is kept.
ArgumentsRaw string `json:"arguments_raw,omitempty"`
ID string `json:"id,omitempty"`
}
// Usage is optional accounting; its absence is a completeness finding.
//
// CacheReadTokens matters more than it looks. An agent harness replays the whole
// transcript every turn, so most of the traffic is re-read context rather than new
// input, and on a long run the cache-read count dwarfs both other numbers.
type Usage struct {
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
}
// Contract is what the call was supposed to produce. Without it the auditor can
// still judge trustworthiness but cannot certify validity, and says so.
type Contract struct {
Task string `json:"task,omitempty"`
OutputSchema map[string]any `json:"output_schema,omitempty"`
RequiredTools []string `json:"required_tools,omitempty"`
RequireGrounding bool `json:"require_grounding,omitempty"`
MustInclude []string `json:"must_include,omitempty"`
MustNotInclude []string `json:"must_not_include,omitempty"`
ExpectedFinishReasons []string `json:"expected_finish_reasons,omitempty"`
ExpectedRefusal *bool `json:"expected_refusal,omitempty"`
}
// Source is retrieved context the response may be grounded in.
type Source struct {
ID string `json:"id,omitempty"`
URI string `json:"uri,omitempty"`
Text string `json:"text,omitempty"`
}
// Integrity is the chain-of-custody record for the call.
type Integrity struct {
// RequestSHA256 is the canonical request hash, as printed by `auditor hash`.
RequestSHA256 string `json:"request_sha256,omitempty"`
}
// LoadBundle reads a bundle from a path, or from stdin when path is "-".
func LoadBundle(path string) (*Bundle, error) {
var r io.Reader
if path == "-" {
r = os.Stdin
} else {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("open bundle %s: %w", path, err)
}
defer func() { _ = f.Close() }()
r = f
}
b, err := DecodeBundle(r)
if err != nil {
return nil, fmt.Errorf("decode bundle %s: %w", path, err)
}
return b, nil
}
// DecodeBundle decodes strictly. An unknown field is an error rather than
// ignored, because a bundle that has drifted from the documented schema is
// exactly the case where silent tolerance would produce a false clean verdict.
func DecodeBundle(r io.Reader) (*Bundle, error) {
dec := json.NewDecoder(r)
dec.UseNumber()
dec.DisallowUnknownFields()
var b Bundle
if err := dec.Decode(&b); err != nil {
return nil, err
}
var extra any
if err := dec.Decode(&extra); !errors.Is(err, io.EOF) {
return nil, errors.New("trailing data after bundle")
}
if err := b.Validate(); err != nil {
return nil, err
}
return &b, nil
}
// Validate rejects bundles that cannot be audited at all. It deliberately does
// not check completeness: a sparse bundle is audited and reported as such.
func (b *Bundle) Validate() error {
if b.Response.empty() {
return errors.New("response is empty: nothing to audit")
}
for i, t := range b.Request.Tools {
if t.Name == "" {
return fmt.Errorf("request.tools[%d]: name is required", i)
}
}
for i, c := range b.Response.ToolCalls {
if c.Name == "" {
return fmt.Errorf("response.tool_calls[%d]: name is required", i)
}
}
for i, m := range b.Request.Messages {
if m.Role == "" {
return fmt.Errorf("request.messages[%d]: role is required", i)
}
}
if n := b.Request.NewFrom; n != nil && (*n < 0 || *n > len(b.Request.Messages)) {
return fmt.Errorf("request.new_from %d is outside 0..%d", *n, len(b.Request.Messages))
}
if b.Integrity != nil && b.Integrity.RequestSHA256 != "" && !isHex64(b.Integrity.RequestSHA256) {
return errors.New("integrity.request_sha256 must be 64 lowercase hex characters")
}
return nil
}
func (r Response) empty() bool {
return r.Content == "" && r.FinishReason == "" && r.Output == nil &&
len(r.ToolCalls) == 0 && !r.Refusal
}
// isHex64 requires lowercase, matching the documented format: the declared hash
// is machine-produced by `auditor hash`, so upper case means a hand-edit.
func isHex64(s string) bool {
if len(s) != 64 || s != strings.ToLower(s) {
return false
}
_, err := hex.DecodeString(s)
return err == nil
}
// CanonicalRequest returns the byte form the request hash is taken over: compact
// JSON of the decoded request, object keys sorted, numbers preserved as written.
func (b *Bundle) CanonicalRequest() ([]byte, error) {
request := b.Request
request.NewFrom = nil
raw, err := json.Marshal(request)
if err != nil {
return nil, fmt.Errorf("canonicalise request: %w", err)
}
return raw, nil
}
// RequestHash returns the hex SHA-256 of CanonicalRequest.
func (b *Bundle) RequestHash() (string, error) {
raw, err := b.CanonicalRequest()
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}