diff --git a/internal/platform/auditjson/auditjson.go b/internal/platform/auditjson/auditjson.go new file mode 100644 index 000000000..4e5f654ed --- /dev/null +++ b/internal/platform/auditjson/auditjson.go @@ -0,0 +1,142 @@ +// Package auditjson stores the exact request and response of a call for +// audit: the message as JSON, with credentials removed and an oversized body +// capped rather than dropped silently. +package auditjson + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json/jsontext" + json "encoding/json/v2" + "strings" + "unicode/utf8" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// MaxBytes is the most of one message that is stored. A larger message keeps +// its first MaxBytes, with its full length and SHA-256. +const MaxBytes = 1 << 20 + +// redacted replaces a credential. +const redacted = "[redacted]" + +// Payload is one stored message. JSON is the message itself when it fits under +// MaxBytes. A message that doesn't fit keeps its first MaxBytes as Text, and +// Length and SHA256 describe the whole of it. +type Payload struct { + JSON jsontext.Value `json:"json,omitzero"` + Text string `json:"text,omitzero"` + Truncated bool `json:"truncated,omitzero"` + Length int64 `json:"length,omitzero"` + SHA256 string `json:"sha256,omitzero"` +} + +// FromProto renders a protobuf message as protojson, with credentials removed +// and the result capped. The message is what core sent or received on the +// wire, never anything a connector resolved internally. +func FromProto(msg proto.Message) Payload { + raw, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(msg) + if err != nil { + return Payload{Text: "unrenderable: " + err.Error()} + } + return FromJSON(raw) +} + +// FromValue encodes v as JSON, with credentials removed and the result capped. +func FromValue(v any) Payload { + raw, err := json.Marshal(v) + if err != nil { + return Payload{Text: "unrenderable: " + err.Error()} + } + return FromJSON(raw) +} + +// FromJSON redacts credentials in raw and caps the result. +func FromJSON(raw jsontext.Value) Payload { + clean := Redact(raw) + if len(clean) <= MaxBytes { + return Payload{JSON: clean} + } + sum := sha256.Sum256(clean) + kept := MaxBytes + for kept > 0 && !utf8.RuneStart(clean[kept]) { + kept-- + } + return Payload{ + Text: string(clean[:kept]), Truncated: true, + Length: int64(len(clean)), SHA256: hex.EncodeToString(sum[:]), + } +} + +// Redact replaces every credential in a JSON value with a marker. A key that +// names a credential is redacted whole; a value that reads as an authorization +// header is redacted wherever it appears. A payload that can't be decoded is +// replaced rather than stored unchanged, because its credentials can't be +// found. +func Redact(raw jsontext.Value) jsontext.Value { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return jsontext.Value(`"[unreadable payload]"`) + } + redactValue(v) + out, err := json.Marshal(v) + if err != nil { + return jsontext.Value(`"[unreadable payload]"`) + } + return out +} + +func redactValue(v any) { + switch x := v.(type) { + case map[string]any: + for key, value := range x { + if sensitiveKey(key) { + x[key] = redacted + continue + } + if s, ok := value.(string); ok && looksSecret(s) { + x[key] = redacted + continue + } + redactValue(value) + } + case []any: + for _, value := range x { + redactValue(value) + } + } +} + +// sensitiveKey reports whether a JSON key names a credential. The match is +// deliberately broad: an audit that keeps a credential is worse than one that +// hides a harmless field. +func sensitiveKey(key string) bool { + k := strings.ToLower(key) + switch { + case strings.Contains(k, "authorization"), + strings.Contains(k, "cookie"), + strings.Contains(k, "password"), + strings.Contains(k, "passwd"), + strings.Contains(k, "secret"), + strings.Contains(k, "privatekey"), + strings.Contains(k, "private_key"), + strings.Contains(k, "credential"), + strings.Contains(k, "apikey"), + strings.Contains(k, "api_key"), + strings.Contains(k, "api-key"): + return true + } + compact := strings.NewReplacer("_", "", "-", "").Replace(k) + return compact == "token" || strings.HasSuffix(compact, "token") +} + +// looksSecret reports whether a value is an authorization header, whatever +// key it arrived under. +func looksSecret(value string) bool { + s := strings.TrimSpace(value) + lower := strings.ToLower(s) + return strings.HasPrefix(lower, "bearer ") || strings.HasPrefix(lower, "basic ") || + strings.Contains(s, "-----BEGIN ") +} diff --git a/internal/platform/auditjson/auditjson_test.go b/internal/platform/auditjson/auditjson_test.go new file mode 100644 index 000000000..21c70be08 --- /dev/null +++ b/internal/platform/auditjson/auditjson_test.go @@ -0,0 +1,80 @@ +package auditjson + +import ( + "encoding/json/jsontext" + json "encoding/json/v2" + "strings" + "testing" +) + +// TestStoredPayloadKeepsCredentialsOut guards the audit rule that a stored +// request or response never carries a credential, wherever it sits in the +// message and however the key is spelled. +func TestStoredPayloadKeepsCredentialsOut(t *testing.T) { + tests := []struct { + name string + in string + secret string + // kept is text that must survive redaction, so a payload isn't + // blanked wholesale. + kept string + }{ + { + name: "authorization header", + in: `{"headers":{"Authorization":"Bearer sk-live-123","Content-Type":"application/json"},"body":"ok"}`, + secret: "sk-live-123", + kept: "application/json", + }, + { + name: "api key field", + in: `{"input":{"api_key":"key-abc","repository":"archie"}}`, + secret: "key-abc", + kept: "archie", + }, + { + name: "nested password", + in: `{"request":{"password":"hunter2"},"target":"db"}`, + secret: "hunter2", + kept: "db", + }, + { + name: "bearer value under an innocuous key", + in: `{"headers":{"X-Custom":"Bearer opaque-456"},"event":"push"}`, + secret: "opaque-456", + kept: "push", + }, + { + name: "token field", + in: `{"access_token":"tok-789","scope":"read"}`, + secret: "tok-789", + kept: "read", + }, + { + name: "camel case token field", + in: `{"accessToken":"camel-123","scope":"write"}`, + secret: "camel-123", + kept: "write", + }, + { + name: "harmless fields survive", + in: `{"prompt_tokens":5,"idempotency_key":"k1","total_tokens":9}`, + secret: "", + kept: "idempotency_key", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := FromJSON(jsontext.Value(tt.in)) + out, err := json.Marshal(p) + if err != nil { + t.Fatalf("marshal stored payload: %v", err) + } + if tt.secret != "" && strings.Contains(string(out), tt.secret) { + t.Fatalf("stored payload contains the credential %q: %s", tt.secret, out) + } + if !strings.Contains(string(out), tt.kept) { + t.Fatalf("stored payload dropped %q: %s", tt.kept, out) + } + }) + } +} diff --git a/internal/platform/pg/migrations/00033_delivery_attempts.sql b/internal/platform/pg/migrations/00033_delivery_attempts.sql new file mode 100644 index 000000000..333eb083d --- /dev/null +++ b/internal/platform/pg/migrations/00033_delivery_attempts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- One row per delivery attempt, so a channel's response and its failures are +-- kept rather than only the last error. Nothing is pruned: the volume is low +-- and the record is what makes a delivery auditable. +CREATE TABLE deliveries.attempts ( + message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE, + attempt int NOT NULL, + at timestamptz NOT NULL DEFAULT now(), + ok boolean NOT NULL, + response text NOT NULL DEFAULT '', + error text NOT NULL DEFAULT '', + duration_ms bigint NOT NULL, + PRIMARY KEY (message_seq, attempt) +); + +-- +goose Down +DROP TABLE deliveries.attempts; diff --git a/modules/connectors/audit.go b/modules/connectors/audit.go new file mode 100644 index 000000000..51b658368 --- /dev/null +++ b/modules/connectors/audit.go @@ -0,0 +1,54 @@ +package connectors + +import ( + "context" + + "google.golang.org/protobuf/proto" + + "github.com/samcharles93/archie-core/internal/platform/auditjson" +) + +// Call is one call core made to a connector or MCP server: the request it sent +// and the response it received, rendered as JSON. Credentials are removed and +// an oversized body is capped, so the record is safe to keep. +type Call struct { + Connector string `json:"connector"` + Method string `json:"method"` + Request auditjson.Payload `json:"request"` + Response auditjson.Payload `json:"response,omitzero"` + Error string `json:"error,omitzero"` +} + +// Recorder keeps each call core makes to a connector. The module that owns the +// record supplies one on the context with WithRecorder; without one nothing is +// recorded. +type Recorder interface { + RecordConnectorCall(ctx context.Context, call Call) +} + +type recorderKey struct{} + +// WithRecorder makes every connector call made with ctx report to r. +func WithRecorder(ctx context.Context, r Recorder) context.Context { + return context.WithValue(ctx, recorderKey{}, r) +} + +// record hands c to the recorder on ctx, if there is one. +func record(ctx context.Context, c Call) { + if r, ok := ctx.Value(recorderKey{}).(Recorder); ok && r != nil { + r.RecordConnectorCall(ctx, c) + } +} + +// recordProto records one protobuf wire call. The response is rendered only +// when the call returned one; a transport error is recorded instead. +func recordProto(ctx context.Context, connector, method string, req, resp proto.Message, err error) { + c := Call{Connector: connector, Method: method, Request: auditjson.FromProto(req)} + switch { + case err != nil: + c.Error = err.Error() + case resp != nil: + c.Response = auditjson.FromProto(resp) + } + record(ctx, c) +} diff --git a/modules/connectors/connector.go b/modules/connectors/connector.go index bd3dd382f..1c375b10a 100644 --- a/modules/connectors/connector.go +++ b/modules/connectors/connector.go @@ -215,7 +215,9 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val if err := validate(schema, input); err != nil { return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err) } - resp, err := c.client.Deliver(ctx, &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}) + req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key} + resp, err := c.client.Deliver(ctx, req) + recordProto(ctx, c.name, "deliver", req, resp, err) if err != nil { return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err) } @@ -264,9 +266,11 @@ func (c *Connector) TargetFacts(ctx context.Context, t Target) (TargetFacts, err if !c.facts { return TargetFacts{}, nil } - resp, err := c.client.TargetFacts(ctx, &pb.TargetFactsRequest{Target: &pb.Target{ + req := &pb.TargetFactsRequest{Target: &pb.Target{ Kind: t.Kind, Id: t.ID, Environment: t.Environment, Disposable: t.Disposable, - }}) + }} + resp, err := c.client.TargetFacts(ctx, req) + recordProto(ctx, c.name, "target_facts", req, resp, err) if err != nil { return TargetFacts{}, fmt.Errorf("connector %q: target facts: %w", c.name, err) } @@ -302,7 +306,9 @@ func (c *Connector) Name() string { return c.name } // Parse turns a payload the caller has already verified into typed events, // each checked against its declared schema. func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]string) ([]Event, error) { - resp, err := c.client.Parse(ctx, &pb.ParseRequest{Body: body, Headers: headers}) + req := &pb.ParseRequest{Body: body, Headers: headers} + resp, err := c.client.Parse(ctx, req) + recordProto(ctx, c.name, "parse", req, resp, err) if err != nil { return nil, fmt.Errorf("connector %q: parse: %w", c.name, err) } @@ -333,7 +339,9 @@ func (c *Connector) Resolve(ctx context.Context, name string, input jsontext.Val if _, err := c.action(name, input); err != nil { return Resolution{}, err } - resp, err := c.client.Resolve(ctx, &pb.ResolveRequest{Action: name, Input: input}) + req := &pb.ResolveRequest{Action: name, Input: input} + resp, err := c.client.Resolve(ctx, req) + recordProto(ctx, c.name, "resolve", req, resp, err) if err != nil { return Resolution{}, fmt.Errorf("connector %q: resolve %q: %w", c.name, name, err) } @@ -357,9 +365,11 @@ func (c *Connector) Invoke(ctx context.Context, inv Invocation) (string, error) } inv.Input = Canonical(inv.Input) digest := Digest(inv.Action, inv.Input) - resp, err := c.client.Invoke(ctx, &pb.InvokeRequest{ + req := &pb.InvokeRequest{ Action: inv.Action, Input: inv.Input, IdempotencyKey: inv.IdempotencyKey, Digest: digest, - }) + } + resp, err := c.client.Invoke(ctx, req) + recordProto(ctx, c.name, "invoke", req, resp, err) if err != nil { return "", fmt.Errorf("connector %q: invoke %q: %w", c.name, inv.Action, err) } @@ -382,7 +392,9 @@ func (c *Connector) Lookup(ctx context.Context, name string, input jsontext.Valu if err := validate(l.input, input); err != nil { return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err) } - resp, err := c.client.Lookup(ctx, &pb.LookupRequest{Lookup: name, Input: input}) + req := &pb.LookupRequest{Lookup: name, Input: input} + resp, err := c.client.Lookup(ctx, req) + recordProto(ctx, c.name, "lookup", req, resp, err) if err != nil { return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err) } diff --git a/modules/connectors/mcp.go b/modules/connectors/mcp.go index 6aa578107..8626991e0 100644 --- a/modules/connectors/mcp.go +++ b/modules/connectors/mcp.go @@ -18,6 +18,7 @@ import ( "github.com/google/jsonschema-go/jsonschema" sdk "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/samcharles93/archie-core/internal/platform/auditjson" "github.com/samcharles93/archie-core/modules/connectors/internal/egress" "github.com/samcharles93/archie-core/modules/connectors/internal/host" ) @@ -182,16 +183,21 @@ func (s *Server) call(ctx context.Context, name, kind string, input jsontext.Val if err != nil { return "", false, fmt.Errorf("mcp server %q: %w", s.name, err) } + request := auditjson.FromValue(map[string]any{"tool": name, "arguments": args}) ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout) defer cancel() res, err := cs.CallTool(ctx, &sdk.CallToolParams{Name: name, Arguments: args}) if err != nil { s.drop(cs, fmt.Errorf("tool %q: %w", name, err)) + record(ctx, Call{Connector: s.name, Method: kind, Request: request, Error: err.Error()}) return "", false, fmt.Errorf("mcp server %q: tool %q: %w", s.name, name, err) } // A tool that answered with an error fails this call, not the run: the // message is the server's own, so a bad query reads as a bad query. out := capResult(render(res)) + record(ctx, Call{Connector: s.name, Method: kind, Request: request, Response: auditjson.FromValue(map[string]any{ + "output": out, "is_error": res.IsError, + })}) if res.IsError { return "", false, fmt.Errorf("mcp server %q: tool %q: %s", s.name, name, out) } diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index d1b0c574b..21bbd4958 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -72,7 +72,8 @@ type ApprovalRequest struct { ExpiresAt time.Time } -// Message is a composer message as the API shows it. +// Message is a composer message as the API shows it. Attempts are the delivery +// attempts of the connector rows that carry the same message, one per channel. type Message struct { Seq int64 `json:"seq"` ID uuid.UUID `json:"id"` @@ -83,6 +84,18 @@ type Message struct { Body string `json:"body"` ExpiresAt *time.Time `json:"expires_at,omitzero"` CreatedAt time.Time `json:"created_at"` + Attempts []Attempt `json:"attempts,omitzero"` +} + +// Attempt is one delivery attempt on one channel. +type Attempt struct { + Channel string `json:"channel"` + Attempt int `json:"attempt"` + At time.Time `json:"at"` + OK bool `json:"ok"` + Response string `json:"response,omitzero"` + Error string `json:"error,omitzero"` + DurationMS int64 `json:"duration_ms"` } // Channels are the running connectors a message can go out on. @@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) { } func (m *Module) deliver(ctx context.Context, msg store.Message) { + start := time.Now() external, err := m.sendTo(ctx, msg) + took := time.Since(start) if err != nil { m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) - if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil { + if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil { m.log.ErrorContext(ctx, "record failed delivery", "error", err) } return } - if err := m.store.Sent(ctx, msg.Seq, external); err != nil { + if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { m.log.ErrorContext(ctx, "record delivery", "error", err) } } @@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") return } + keys := make([]string, 0, len(rows)) + for _, row := range rows { + keys = append(keys, row.Key) + } + attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys) + if err != nil { + m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") + return + } + byKey := make(map[string][]Attempt, len(rows)) + for _, a := range attempts { + byKey[a.Key] = append(byKey[a.Key], Attempt{ + Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK, + Response: a.Response, Error: a.Error, DurationMS: a.DurationMS, + }) + } out := make([]Message, 0, len(rows)) for _, row := range rows { - msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt} + msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]} if row.RunID.Valid { msg.RunID = &row.RunID.UUID } diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index b95ca7838..43622d3dc 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat return m, true, nil } -// Sent records that a channel confirmed a message. -func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, - seq, externalID); err != nil { - return fmt.Errorf("mark message sent: %w", err) - } - return nil +// Attempt is one delivery attempt as the dashboard shows it. +type Attempt struct { + // Key is the message key the attempt's channel row shares with the + // composer message. + Key string `db:"key"` + Channel string `db:"channel"` + MessageSeq int64 `db:"message_seq"` + Attempt int `db:"attempt"` + At time.Time `db:"at"` + OK bool `db:"ok"` + Response string `db:"response"` + Error string `db:"error"` + DurationMS int64 `db:"duration_ms"` +} + +// Sent records that a channel confirmed a message, with the attempt that +// confirmed it. +func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took}, + `UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID) } // SentBy finds a message its channel confirmed under one of ids. The run is @@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) return run, true, nil } -// Failed records why an attempt failed; the lease already set when it is -// tried again. -func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil { - return fmt.Errorf("record failed delivery: %w", err) +// Failed records why an attempt failed, with the attempt that failed; the +// lease already set when it is tried again. +func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, + `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) +} + +// attemptOutcome is one attempt's result. +type attemptOutcome struct { + ok bool + response string + failure string + took time.Duration +} + +// attempt marks the message row and stores the attempt in one transaction, so +// a retried message never loses the reason it was retried. +func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil { + return fmt.Errorf("record delivery: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms) + VALUES ($1, $2, $3, $4, $5, $6)`, + seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) } return nil } + +// AttemptsFor returns the attempts of every connector delivery carrying one of +// keys. Composer rows have none: they are stored already sent. +func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) { + if len(keys) == 0 { + return nil, nil + } + var out []Attempt + if err := s.db.SelectContext(ctx, &out, ` + SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms + FROM deliveries.attempts a + JOIN deliveries.messages m ON m.seq = a.message_seq + WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer' + ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil { + return nil, fmt.Errorf("list delivery attempts: %w", err) + } + return out, nil +} diff --git a/modules/judgement/internal/store/store.go b/modules/judgement/internal/store/store.go index c6468170a..abd9407d3 100644 --- a/modules/judgement/internal/store/store.go +++ b/modules/judgement/internal/store/store.go @@ -60,6 +60,20 @@ func (s *Store) Record(ctx context.Context, d Decision) (Decision, error) { return kept, nil } +// AppendCall adds one connector call to a decision body's calls. The request +// and response therefore sit with the judgement they justified, whether they +// were part of judging the action or of running it. +func (s *Store) AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error { + if _, err := s.db.ExecContext(ctx, ` + UPDATE judgement.decisions + SET body = jsonb_set(body::jsonb, '{calls}', + COALESCE(body::jsonb->'calls', '[]'::jsonb) || jsonb_build_array($3::jsonb))::json + WHERE org_id = $1 AND id = $2`, orgID, id, call); err != nil { + return fmt.Errorf("record connector call: %w", err) + } + return nil +} + // ForRun returns a run's decisions in the order they were made. func (s *Store) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]Decision, error) { var out []Decision diff --git a/modules/judgement/judgement.go b/modules/judgement/judgement.go index a69c067da..5b089b5aa 100644 --- a/modules/judgement/judgement.go +++ b/modules/judgement/judgement.go @@ -157,6 +157,11 @@ type Decision struct { // Dry marks a decision made for a dry run. It is recorded for the run's // trail and counts toward nothing: not spend, not a class's history. Dry bool `json:"dry,omitzero"` + // Calls are the calls to the connector behind this decision: the resolve + // that placed its target, the target facts it was judged on, and the + // invoke that ran it. Each holds the request core sent and the response it + // received, with credentials removed. + Calls []connectors.Call `json:"calls,omitzero"` } // Owner is the id of the person who answers for the decision's target, or empty @@ -208,6 +213,29 @@ type FailureRecorder interface { type recorder interface { Record(ctx context.Context, d store.Decision) (store.Decision, error) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]store.Decision, error) + AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error +} + +// callCollector keeps the connector calls one judgement made, so the decision +// body records what core sent and received. +type callCollector struct { + mu sync.Mutex + calls []connectors.Call +} + +// RecordConnectorCall keeps one call. Guards judge in parallel, so the +// collector is safe for concurrent use. +func (c *callCollector) RecordConnectorCall(_ context.Context, call connectors.Call) { + c.mu.Lock() + defer c.mu.Unlock() + c.calls = append(c.calls, call) +} + +// take returns the calls collected so far. +func (c *callCollector) take() []connectors.Call { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.calls) } // Module judges one organisation's actions. @@ -249,6 +277,8 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) { if !ok { return Decision{}, fmt.Errorf("%q isn't running", req.Connector) } + calls := &callCollector{} + ctx = connectors.WithRecorder(ctx, calls) res, err := tool.Resolve(ctx, req.Action, req.Input) if err != nil { return Decision{}, fmt.Errorf("resolve %s %s: %w", req.Connector, req.Action, err) @@ -259,6 +289,7 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) { } d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence, Overlay: req.Overlay}), a) d.ID, d.RunID, d.Key, d.At, d.Dry = uuid.New(), req.RunID, req.Key, m.now().UTC(), req.Dry + d.Calls = calls.take() return m.record(ctx, d) } @@ -361,7 +392,10 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k if !ok { return "", fmt.Errorf("%q isn't running", d.Action.Connector) } + calls := &callCollector{} + ctx = connectors.WithRecorder(ctx, calls) out, err := tool.Invoke(ctx, connectors.Invocation{Action: d.Action.Name, Input: input, IdempotencyKey: key}) + m.keepCalls(ctx, d.ID, calls.take()) // A shutdown cancels the call without the action having gone wrong, and a // dry run's failure is not evidence about the class. if err != nil && m.failed != nil && !d.Dry && !errors.Is(err, context.Canceled) { @@ -372,6 +406,25 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k return out, err } +// keepCalls adds the connector calls one invoke made to its decision body, so +// the executed action's request and response sit with the judgement that let +// it run. +func (m *Module) keepCalls(ctx context.Context, id uuid.UUID, calls []connectors.Call) { + if id == uuid.Nil { + return + } + for _, call := range calls { + raw, err := json.Marshal(call) + if err != nil { + m.log.ErrorContext(ctx, "encode connector call", "decision", id, "error", err) + continue + } + if err := m.store.AppendCall(ctx, m.orgID, id, string(raw)); err != nil { + m.log.ErrorContext(ctx, "record connector call", "decision", id, "error", err) + } + } +} + // Approval is a person's permission to run one decision's exact action. It // names the decision and the action's digest, so it covers that action and // nothing else, and it says who gave it and when. diff --git a/modules/runs/trail.go b/modules/runs/trail.go index 7880576e3..d3977b880 100644 --- a/modules/runs/trail.go +++ b/modules/runs/trail.go @@ -101,11 +101,16 @@ type TrailEntry struct { Reasoning string `json:"reasoning,omitzero"` ToolCalls []ToolCallView `json:"tool_calls,omitzero"` Usage *chat.Usage `json:"usage,omitzero"` - Input string `json:"input,omitzero"` - Output string `json:"output,omitzero"` - Value jsontext.Value `json:"value,omitzero"` - Error string `json:"error,omitzero"` - Unknown bool `json:"unknown,omitzero"` + // Request is the model request exactly as it was sent: the messages, the + // tools offered and the sampling settings. + Request jsontext.Value `json:"request,omitzero"` + // FinishReason is the wire finish reason of a model call. + FinishReason string `json:"finish_reason,omitzero"` + Input string `json:"input,omitzero"` + Output string `json:"output,omitzero"` + Value jsontext.Value `json:"value,omitzero"` + Error string `json:"error,omitzero"` + Unknown bool `json:"unknown,omitzero"` } // ToolCallView is a tool call a model asked for. @@ -331,6 +336,7 @@ func entry(s store.Recorded) TrailEntry { if je.Kind == agentloop.EntryCompact { e.Kind = "compaction" } + e.Request, e.FinishReason = je.Request, je.FinishReason usage := je.Usage e.Usage = &usage if je.Message != nil { diff --git a/ui/src/archie/ActionTrailView.vue b/ui/src/archie/ActionTrailView.vue index 0532954f0..bb6a87d2a 100644 --- a/ui/src/archie/ActionTrailView.vue +++ b/ui/src/archie/ActionTrailView.vue @@ -53,6 +53,26 @@ const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary"); {{ json(decision.action.input) }} +
+ Connector calls +
+
+ {{ c.connector }} {{ c.method }} + Request + + {{ json(c.request) }} + + + {{ c.error }} +
+
+
+