From 4b97ac58415a0404464c5c9a7caf5a95832e707a Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 12:04:36 +1100 Subject: [PATCH 1/3] feat(audit): a connector call keeps the request core sent and the response it received Resolve, target facts, invoke, lookup, deliver and parse record the protobuf wire messages as JSON on the context recorder the owning module supplies. Credentials are redacted and a body over 1 MiB keeps its prefix, full length and SHA-256. Judgement stores the resolve and target facts in the decision body and appends the executed invoke to it. --- internal/platform/auditjson/auditjson.go | 142 ++++++++++++++++++ internal/platform/auditjson/auditjson_test.go | 80 ++++++++++ modules/connectors/audit.go | 54 +++++++ modules/connectors/connector.go | 28 +++- modules/connectors/mcp.go | 6 + modules/judgement/internal/store/store.go | 14 ++ modules/judgement/judgement.go | 53 +++++++ 7 files changed, 369 insertions(+), 8 deletions(-) create mode 100644 internal/platform/auditjson/auditjson.go create mode 100644 internal/platform/auditjson/auditjson_test.go create mode 100644 modules/connectors/audit.go diff --git a/internal/platform/auditjson/auditjson.go b/internal/platform/auditjson/auditjson.go new file mode 100644 index 000000000..4e5f654ed --- /dev/null +++ b/internal/platform/auditjson/auditjson.go @@ -0,0 +1,142 @@ +// Package auditjson stores the exact request and response of a call for +// audit: the message as JSON, with credentials removed and an oversized body +// capped rather than dropped silently. +package auditjson + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json/jsontext" + json "encoding/json/v2" + "strings" + "unicode/utf8" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// MaxBytes is the most of one message that is stored. A larger message keeps +// its first MaxBytes, with its full length and SHA-256. +const MaxBytes = 1 << 20 + +// redacted replaces a credential. +const redacted = "[redacted]" + +// Payload is one stored message. JSON is the message itself when it fits under +// MaxBytes. A message that doesn't fit keeps its first MaxBytes as Text, and +// Length and SHA256 describe the whole of it. +type Payload struct { + JSON jsontext.Value `json:"json,omitzero"` + Text string `json:"text,omitzero"` + Truncated bool `json:"truncated,omitzero"` + Length int64 `json:"length,omitzero"` + SHA256 string `json:"sha256,omitzero"` +} + +// FromProto renders a protobuf message as protojson, with credentials removed +// and the result capped. The message is what core sent or received on the +// wire, never anything a connector resolved internally. +func FromProto(msg proto.Message) Payload { + raw, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(msg) + if err != nil { + return Payload{Text: "unrenderable: " + err.Error()} + } + return FromJSON(raw) +} + +// FromValue encodes v as JSON, with credentials removed and the result capped. +func FromValue(v any) Payload { + raw, err := json.Marshal(v) + if err != nil { + return Payload{Text: "unrenderable: " + err.Error()} + } + return FromJSON(raw) +} + +// FromJSON redacts credentials in raw and caps the result. +func FromJSON(raw jsontext.Value) Payload { + clean := Redact(raw) + if len(clean) <= MaxBytes { + return Payload{JSON: clean} + } + sum := sha256.Sum256(clean) + kept := MaxBytes + for kept > 0 && !utf8.RuneStart(clean[kept]) { + kept-- + } + return Payload{ + Text: string(clean[:kept]), Truncated: true, + Length: int64(len(clean)), SHA256: hex.EncodeToString(sum[:]), + } +} + +// Redact replaces every credential in a JSON value with a marker. A key that +// names a credential is redacted whole; a value that reads as an authorization +// header is redacted wherever it appears. A payload that can't be decoded is +// replaced rather than stored unchanged, because its credentials can't be +// found. +func Redact(raw jsontext.Value) jsontext.Value { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return jsontext.Value(`"[unreadable payload]"`) + } + redactValue(v) + out, err := json.Marshal(v) + if err != nil { + return jsontext.Value(`"[unreadable payload]"`) + } + return out +} + +func redactValue(v any) { + switch x := v.(type) { + case map[string]any: + for key, value := range x { + if sensitiveKey(key) { + x[key] = redacted + continue + } + if s, ok := value.(string); ok && looksSecret(s) { + x[key] = redacted + continue + } + redactValue(value) + } + case []any: + for _, value := range x { + redactValue(value) + } + } +} + +// sensitiveKey reports whether a JSON key names a credential. The match is +// deliberately broad: an audit that keeps a credential is worse than one that +// hides a harmless field. +func sensitiveKey(key string) bool { + k := strings.ToLower(key) + switch { + case strings.Contains(k, "authorization"), + strings.Contains(k, "cookie"), + strings.Contains(k, "password"), + strings.Contains(k, "passwd"), + strings.Contains(k, "secret"), + strings.Contains(k, "privatekey"), + strings.Contains(k, "private_key"), + strings.Contains(k, "credential"), + strings.Contains(k, "apikey"), + strings.Contains(k, "api_key"), + strings.Contains(k, "api-key"): + return true + } + compact := strings.NewReplacer("_", "", "-", "").Replace(k) + return compact == "token" || strings.HasSuffix(compact, "token") +} + +// looksSecret reports whether a value is an authorization header, whatever +// key it arrived under. +func looksSecret(value string) bool { + s := strings.TrimSpace(value) + lower := strings.ToLower(s) + return strings.HasPrefix(lower, "bearer ") || strings.HasPrefix(lower, "basic ") || + strings.Contains(s, "-----BEGIN ") +} diff --git a/internal/platform/auditjson/auditjson_test.go b/internal/platform/auditjson/auditjson_test.go new file mode 100644 index 000000000..21c70be08 --- /dev/null +++ b/internal/platform/auditjson/auditjson_test.go @@ -0,0 +1,80 @@ +package auditjson + +import ( + "encoding/json/jsontext" + json "encoding/json/v2" + "strings" + "testing" +) + +// TestStoredPayloadKeepsCredentialsOut guards the audit rule that a stored +// request or response never carries a credential, wherever it sits in the +// message and however the key is spelled. +func TestStoredPayloadKeepsCredentialsOut(t *testing.T) { + tests := []struct { + name string + in string + secret string + // kept is text that must survive redaction, so a payload isn't + // blanked wholesale. + kept string + }{ + { + name: "authorization header", + in: `{"headers":{"Authorization":"Bearer sk-live-123","Content-Type":"application/json"},"body":"ok"}`, + secret: "sk-live-123", + kept: "application/json", + }, + { + name: "api key field", + in: `{"input":{"api_key":"key-abc","repository":"archie"}}`, + secret: "key-abc", + kept: "archie", + }, + { + name: "nested password", + in: `{"request":{"password":"hunter2"},"target":"db"}`, + secret: "hunter2", + kept: "db", + }, + { + name: "bearer value under an innocuous key", + in: `{"headers":{"X-Custom":"Bearer opaque-456"},"event":"push"}`, + secret: "opaque-456", + kept: "push", + }, + { + name: "token field", + in: `{"access_token":"tok-789","scope":"read"}`, + secret: "tok-789", + kept: "read", + }, + { + name: "camel case token field", + in: `{"accessToken":"camel-123","scope":"write"}`, + secret: "camel-123", + kept: "write", + }, + { + name: "harmless fields survive", + in: `{"prompt_tokens":5,"idempotency_key":"k1","total_tokens":9}`, + secret: "", + kept: "idempotency_key", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := FromJSON(jsontext.Value(tt.in)) + out, err := json.Marshal(p) + if err != nil { + t.Fatalf("marshal stored payload: %v", err) + } + if tt.secret != "" && strings.Contains(string(out), tt.secret) { + t.Fatalf("stored payload contains the credential %q: %s", tt.secret, out) + } + if !strings.Contains(string(out), tt.kept) { + t.Fatalf("stored payload dropped %q: %s", tt.kept, out) + } + }) + } +} diff --git a/modules/connectors/audit.go b/modules/connectors/audit.go new file mode 100644 index 000000000..51b658368 --- /dev/null +++ b/modules/connectors/audit.go @@ -0,0 +1,54 @@ +package connectors + +import ( + "context" + + "google.golang.org/protobuf/proto" + + "github.com/samcharles93/archie-core/internal/platform/auditjson" +) + +// Call is one call core made to a connector or MCP server: the request it sent +// and the response it received, rendered as JSON. Credentials are removed and +// an oversized body is capped, so the record is safe to keep. +type Call struct { + Connector string `json:"connector"` + Method string `json:"method"` + Request auditjson.Payload `json:"request"` + Response auditjson.Payload `json:"response,omitzero"` + Error string `json:"error,omitzero"` +} + +// Recorder keeps each call core makes to a connector. The module that owns the +// record supplies one on the context with WithRecorder; without one nothing is +// recorded. +type Recorder interface { + RecordConnectorCall(ctx context.Context, call Call) +} + +type recorderKey struct{} + +// WithRecorder makes every connector call made with ctx report to r. +func WithRecorder(ctx context.Context, r Recorder) context.Context { + return context.WithValue(ctx, recorderKey{}, r) +} + +// record hands c to the recorder on ctx, if there is one. +func record(ctx context.Context, c Call) { + if r, ok := ctx.Value(recorderKey{}).(Recorder); ok && r != nil { + r.RecordConnectorCall(ctx, c) + } +} + +// recordProto records one protobuf wire call. The response is rendered only +// when the call returned one; a transport error is recorded instead. +func recordProto(ctx context.Context, connector, method string, req, resp proto.Message, err error) { + c := Call{Connector: connector, Method: method, Request: auditjson.FromProto(req)} + switch { + case err != nil: + c.Error = err.Error() + case resp != nil: + c.Response = auditjson.FromProto(resp) + } + record(ctx, c) +} diff --git a/modules/connectors/connector.go b/modules/connectors/connector.go index bd3dd382f..1c375b10a 100644 --- a/modules/connectors/connector.go +++ b/modules/connectors/connector.go @@ -215,7 +215,9 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val if err := validate(schema, input); err != nil { return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err) } - resp, err := c.client.Deliver(ctx, &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}) + req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key} + resp, err := c.client.Deliver(ctx, req) + recordProto(ctx, c.name, "deliver", req, resp, err) if err != nil { return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err) } @@ -264,9 +266,11 @@ func (c *Connector) TargetFacts(ctx context.Context, t Target) (TargetFacts, err if !c.facts { return TargetFacts{}, nil } - resp, err := c.client.TargetFacts(ctx, &pb.TargetFactsRequest{Target: &pb.Target{ + req := &pb.TargetFactsRequest{Target: &pb.Target{ Kind: t.Kind, Id: t.ID, Environment: t.Environment, Disposable: t.Disposable, - }}) + }} + resp, err := c.client.TargetFacts(ctx, req) + recordProto(ctx, c.name, "target_facts", req, resp, err) if err != nil { return TargetFacts{}, fmt.Errorf("connector %q: target facts: %w", c.name, err) } @@ -302,7 +306,9 @@ func (c *Connector) Name() string { return c.name } // Parse turns a payload the caller has already verified into typed events, // each checked against its declared schema. func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]string) ([]Event, error) { - resp, err := c.client.Parse(ctx, &pb.ParseRequest{Body: body, Headers: headers}) + req := &pb.ParseRequest{Body: body, Headers: headers} + resp, err := c.client.Parse(ctx, req) + recordProto(ctx, c.name, "parse", req, resp, err) if err != nil { return nil, fmt.Errorf("connector %q: parse: %w", c.name, err) } @@ -333,7 +339,9 @@ func (c *Connector) Resolve(ctx context.Context, name string, input jsontext.Val if _, err := c.action(name, input); err != nil { return Resolution{}, err } - resp, err := c.client.Resolve(ctx, &pb.ResolveRequest{Action: name, Input: input}) + req := &pb.ResolveRequest{Action: name, Input: input} + resp, err := c.client.Resolve(ctx, req) + recordProto(ctx, c.name, "resolve", req, resp, err) if err != nil { return Resolution{}, fmt.Errorf("connector %q: resolve %q: %w", c.name, name, err) } @@ -357,9 +365,11 @@ func (c *Connector) Invoke(ctx context.Context, inv Invocation) (string, error) } inv.Input = Canonical(inv.Input) digest := Digest(inv.Action, inv.Input) - resp, err := c.client.Invoke(ctx, &pb.InvokeRequest{ + req := &pb.InvokeRequest{ Action: inv.Action, Input: inv.Input, IdempotencyKey: inv.IdempotencyKey, Digest: digest, - }) + } + resp, err := c.client.Invoke(ctx, req) + recordProto(ctx, c.name, "invoke", req, resp, err) if err != nil { return "", fmt.Errorf("connector %q: invoke %q: %w", c.name, inv.Action, err) } @@ -382,7 +392,9 @@ func (c *Connector) Lookup(ctx context.Context, name string, input jsontext.Valu if err := validate(l.input, input); err != nil { return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err) } - resp, err := c.client.Lookup(ctx, &pb.LookupRequest{Lookup: name, Input: input}) + req := &pb.LookupRequest{Lookup: name, Input: input} + resp, err := c.client.Lookup(ctx, req) + recordProto(ctx, c.name, "lookup", req, resp, err) if err != nil { return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err) } diff --git a/modules/connectors/mcp.go b/modules/connectors/mcp.go index 6aa578107..8626991e0 100644 --- a/modules/connectors/mcp.go +++ b/modules/connectors/mcp.go @@ -18,6 +18,7 @@ import ( "github.com/google/jsonschema-go/jsonschema" sdk "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/samcharles93/archie-core/internal/platform/auditjson" "github.com/samcharles93/archie-core/modules/connectors/internal/egress" "github.com/samcharles93/archie-core/modules/connectors/internal/host" ) @@ -182,16 +183,21 @@ func (s *Server) call(ctx context.Context, name, kind string, input jsontext.Val if err != nil { return "", false, fmt.Errorf("mcp server %q: %w", s.name, err) } + request := auditjson.FromValue(map[string]any{"tool": name, "arguments": args}) ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout) defer cancel() res, err := cs.CallTool(ctx, &sdk.CallToolParams{Name: name, Arguments: args}) if err != nil { s.drop(cs, fmt.Errorf("tool %q: %w", name, err)) + record(ctx, Call{Connector: s.name, Method: kind, Request: request, Error: err.Error()}) return "", false, fmt.Errorf("mcp server %q: tool %q: %w", s.name, name, err) } // A tool that answered with an error fails this call, not the run: the // message is the server's own, so a bad query reads as a bad query. out := capResult(render(res)) + record(ctx, Call{Connector: s.name, Method: kind, Request: request, Response: auditjson.FromValue(map[string]any{ + "output": out, "is_error": res.IsError, + })}) if res.IsError { return "", false, fmt.Errorf("mcp server %q: tool %q: %s", s.name, name, out) } diff --git a/modules/judgement/internal/store/store.go b/modules/judgement/internal/store/store.go index c6468170a..abd9407d3 100644 --- a/modules/judgement/internal/store/store.go +++ b/modules/judgement/internal/store/store.go @@ -60,6 +60,20 @@ func (s *Store) Record(ctx context.Context, d Decision) (Decision, error) { return kept, nil } +// AppendCall adds one connector call to a decision body's calls. The request +// and response therefore sit with the judgement they justified, whether they +// were part of judging the action or of running it. +func (s *Store) AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error { + if _, err := s.db.ExecContext(ctx, ` + UPDATE judgement.decisions + SET body = jsonb_set(body::jsonb, '{calls}', + COALESCE(body::jsonb->'calls', '[]'::jsonb) || jsonb_build_array($3::jsonb))::json + WHERE org_id = $1 AND id = $2`, orgID, id, call); err != nil { + return fmt.Errorf("record connector call: %w", err) + } + return nil +} + // ForRun returns a run's decisions in the order they were made. func (s *Store) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]Decision, error) { var out []Decision diff --git a/modules/judgement/judgement.go b/modules/judgement/judgement.go index 3430301f6..e8546e283 100644 --- a/modules/judgement/judgement.go +++ b/modules/judgement/judgement.go @@ -150,6 +150,11 @@ type Decision struct { // Dry marks a decision made for a dry run. It is recorded for the run's // trail and counts toward nothing: not spend, not a class's history. Dry bool `json:"dry,omitzero"` + // Calls are the calls to the connector behind this decision: the resolve + // that placed its target, the target facts it was judged on, and the + // invoke that ran it. Each holds the request core sent and the response it + // received, with credentials removed. + Calls []connectors.Call `json:"calls,omitzero"` } // Owner is the id of the person who answers for the decision's target, or empty @@ -201,6 +206,29 @@ type FailureRecorder interface { type recorder interface { Record(ctx context.Context, d store.Decision) (store.Decision, error) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]store.Decision, error) + AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error +} + +// callCollector keeps the connector calls one judgement made, so the decision +// body records what core sent and received. +type callCollector struct { + mu sync.Mutex + calls []connectors.Call +} + +// RecordConnectorCall keeps one call. Guards judge in parallel, so the +// collector is safe for concurrent use. +func (c *callCollector) RecordConnectorCall(_ context.Context, call connectors.Call) { + c.mu.Lock() + defer c.mu.Unlock() + c.calls = append(c.calls, call) +} + +// take returns the calls collected so far. +func (c *callCollector) take() []connectors.Call { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.calls) } // Module judges one organisation's actions. @@ -242,6 +270,8 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) { if !ok { return Decision{}, fmt.Errorf("%q isn't running", req.Connector) } + calls := &callCollector{} + ctx = connectors.WithRecorder(ctx, calls) res, err := tool.Resolve(ctx, req.Action, req.Input) if err != nil { return Decision{}, fmt.Errorf("resolve %s %s: %w", req.Connector, req.Action, err) @@ -252,6 +282,7 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) { } d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence, Overlay: req.Overlay}), a) d.ID, d.RunID, d.Key, d.At, d.Dry = uuid.New(), req.RunID, req.Key, m.now().UTC(), req.Dry + d.Calls = calls.take() return m.record(ctx, d) } @@ -354,7 +385,10 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k if !ok { return "", fmt.Errorf("%q isn't running", d.Action.Connector) } + calls := &callCollector{} + ctx = connectors.WithRecorder(ctx, calls) out, err := tool.Invoke(ctx, connectors.Invocation{Action: d.Action.Name, Input: input, IdempotencyKey: key}) + m.keepCalls(ctx, d.ID, calls.take()) // A shutdown cancels the call without the action having gone wrong, and a // dry run's failure is not evidence about the class. if err != nil && m.failed != nil && !d.Dry && !errors.Is(err, context.Canceled) { @@ -365,6 +399,25 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k return out, err } +// keepCalls adds the connector calls one invoke made to its decision body, so +// the executed action's request and response sit with the judgement that let +// it run. +func (m *Module) keepCalls(ctx context.Context, id uuid.UUID, calls []connectors.Call) { + if id == uuid.Nil { + return + } + for _, call := range calls { + raw, err := json.Marshal(call) + if err != nil { + m.log.ErrorContext(ctx, "encode connector call", "decision", id, "error", err) + continue + } + if err := m.store.AppendCall(ctx, m.orgID, id, string(raw)); err != nil { + m.log.ErrorContext(ctx, "record connector call", "decision", id, "error", err) + } + } +} + // Approval is a person's permission to run one decision's exact action. It // names the decision and the action's digest, so it covers that action and // nothing else, and it says who gave it and when. From db36d853cedd106ca786a04794f0779738289a23 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 12:04:39 +1100 Subject: [PATCH 2/3] feat(deliveries): every attempt keeps its response, its error and how long it took A delivery.attempts row is written with the success or failure the store already records. The composer API returns the connector attempts that carry each message, so the dashboard can show each attempt's outcome and response. --- .../pg/migrations/00033_delivery_attempts.sql | 17 ++++ modules/deliveries/deliveries.go | 40 ++++++++- modules/deliveries/internal/store/store.go | 86 ++++++++++++++++--- 3 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 internal/platform/pg/migrations/00033_delivery_attempts.sql diff --git a/internal/platform/pg/migrations/00033_delivery_attempts.sql b/internal/platform/pg/migrations/00033_delivery_attempts.sql new file mode 100644 index 000000000..333eb083d --- /dev/null +++ b/internal/platform/pg/migrations/00033_delivery_attempts.sql @@ -0,0 +1,17 @@ +-- +goose Up +-- One row per delivery attempt, so a channel's response and its failures are +-- kept rather than only the last error. Nothing is pruned: the volume is low +-- and the record is what makes a delivery auditable. +CREATE TABLE deliveries.attempts ( + message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE, + attempt int NOT NULL, + at timestamptz NOT NULL DEFAULT now(), + ok boolean NOT NULL, + response text NOT NULL DEFAULT '', + error text NOT NULL DEFAULT '', + duration_ms bigint NOT NULL, + PRIMARY KEY (message_seq, attempt) +); + +-- +goose Down +DROP TABLE deliveries.attempts; diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index d1b0c574b..21bbd4958 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -72,7 +72,8 @@ type ApprovalRequest struct { ExpiresAt time.Time } -// Message is a composer message as the API shows it. +// Message is a composer message as the API shows it. Attempts are the delivery +// attempts of the connector rows that carry the same message, one per channel. type Message struct { Seq int64 `json:"seq"` ID uuid.UUID `json:"id"` @@ -83,6 +84,18 @@ type Message struct { Body string `json:"body"` ExpiresAt *time.Time `json:"expires_at,omitzero"` CreatedAt time.Time `json:"created_at"` + Attempts []Attempt `json:"attempts,omitzero"` +} + +// Attempt is one delivery attempt on one channel. +type Attempt struct { + Channel string `json:"channel"` + Attempt int `json:"attempt"` + At time.Time `json:"at"` + OK bool `json:"ok"` + Response string `json:"response,omitzero"` + Error string `json:"error,omitzero"` + DurationMS int64 `json:"duration_ms"` } // Channels are the running connectors a message can go out on. @@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) { } func (m *Module) deliver(ctx context.Context, msg store.Message) { + start := time.Now() external, err := m.sendTo(ctx, msg) + took := time.Since(start) if err != nil { m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) - if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil { + if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil { m.log.ErrorContext(ctx, "record failed delivery", "error", err) } return } - if err := m.store.Sent(ctx, msg.Seq, external); err != nil { + if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil { m.log.ErrorContext(ctx, "record delivery", "error", err) } } @@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) { httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") return } + keys := make([]string, 0, len(rows)) + for _, row := range rows { + keys = append(keys, row.Key) + } + attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys) + if err != nil { + m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err) + httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages") + return + } + byKey := make(map[string][]Attempt, len(rows)) + for _, a := range attempts { + byKey[a.Key] = append(byKey[a.Key], Attempt{ + Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK, + Response: a.Response, Error: a.Error, DurationMS: a.DurationMS, + }) + } out := make([]Message, 0, len(rows)) for _, row := range rows { - msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt} + msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]} if row.RunID.Valid { msg.RunID = &row.RunID.UUID } diff --git a/modules/deliveries/internal/store/store.go b/modules/deliveries/internal/store/store.go index b95ca7838..43622d3dc 100644 --- a/modules/deliveries/internal/store/store.go +++ b/modules/deliveries/internal/store/store.go @@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat return m, true, nil } -// Sent records that a channel confirmed a message. -func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, - seq, externalID); err != nil { - return fmt.Errorf("mark message sent: %w", err) - } - return nil +// Attempt is one delivery attempt as the dashboard shows it. +type Attempt struct { + // Key is the message key the attempt's channel row shares with the + // composer message. + Key string `db:"key"` + Channel string `db:"channel"` + MessageSeq int64 `db:"message_seq"` + Attempt int `db:"attempt"` + At time.Time `db:"at"` + OK bool `db:"ok"` + Response string `db:"response"` + Error string `db:"error"` + DurationMS int64 `db:"duration_ms"` +} + +// Sent records that a channel confirmed a message, with the attempt that +// confirmed it. +func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took}, + `UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID) } // SentBy finds a message its channel confirmed under one of ids. The run is @@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string) return run, true, nil } -// Failed records why an attempt failed; the lease already set when it is -// tried again. -func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { - if _, err := s.db.ExecContext(ctx, ` - UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil { - return fmt.Errorf("record failed delivery: %w", err) +// Failed records why an attempt failed, with the attempt that failed; the +// lease already set when it is tried again. +func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error { + return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took}, + `UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason) +} + +// attemptOutcome is one attempt's result. +type attemptOutcome struct { + ok bool + response string + failure string + took time.Duration +} + +// attempt marks the message row and stores the attempt in one transaction, so +// a retried message never loses the reason it was retried. +func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error { + tx, err := s.db.BeginTxx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil { + return fmt.Errorf("record delivery: %w", err) + } + if _, err := tx.ExecContext(ctx, ` + INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms) + VALUES ($1, $2, $3, $4, $5, $6)`, + seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("record delivery attempt: %w", err) } return nil } + +// AttemptsFor returns the attempts of every connector delivery carrying one of +// keys. Composer rows have none: they are stored already sent. +func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) { + if len(keys) == 0 { + return nil, nil + } + var out []Attempt + if err := s.db.SelectContext(ctx, &out, ` + SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms + FROM deliveries.attempts a + JOIN deliveries.messages m ON m.seq = a.message_seq + WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer' + ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil { + return nil, fmt.Errorf("list delivery attempts: %w", err) + } + return out, nil +} From 46372b51d5e41a092c386c963954ae1ebdd85f34 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 12:04:41 +1100 Subject: [PATCH 3/3] feat(runs,ui): the trail shows each model request and response, connector calls and delivery attempts The run trail exposes the model request the journal recorded and its finish reason. Each model and tool step opens a Request and a Response, and each decision lists the connector calls behind it. The composer lists each delivery attempt with its outcome and response. --- modules/runs/trail.go | 16 ++++++--- ui/src/archie/ActionTrailView.vue | 20 +++++++++++ ui/src/archie/ComposerView.vue | 15 ++++++++ ui/src/archie/RunView.vue | 57 ++++++++++++++++++++++++------- ui/src/archie/api.ts | 25 ++++++++++++++ 5 files changed, 115 insertions(+), 18 deletions(-) diff --git a/modules/runs/trail.go b/modules/runs/trail.go index 7880576e3..d3977b880 100644 --- a/modules/runs/trail.go +++ b/modules/runs/trail.go @@ -101,11 +101,16 @@ type TrailEntry struct { Reasoning string `json:"reasoning,omitzero"` ToolCalls []ToolCallView `json:"tool_calls,omitzero"` Usage *chat.Usage `json:"usage,omitzero"` - Input string `json:"input,omitzero"` - Output string `json:"output,omitzero"` - Value jsontext.Value `json:"value,omitzero"` - Error string `json:"error,omitzero"` - Unknown bool `json:"unknown,omitzero"` + // Request is the model request exactly as it was sent: the messages, the + // tools offered and the sampling settings. + Request jsontext.Value `json:"request,omitzero"` + // FinishReason is the wire finish reason of a model call. + FinishReason string `json:"finish_reason,omitzero"` + Input string `json:"input,omitzero"` + Output string `json:"output,omitzero"` + Value jsontext.Value `json:"value,omitzero"` + Error string `json:"error,omitzero"` + Unknown bool `json:"unknown,omitzero"` } // ToolCallView is a tool call a model asked for. @@ -331,6 +336,7 @@ func entry(s store.Recorded) TrailEntry { if je.Kind == agentloop.EntryCompact { e.Kind = "compaction" } + e.Request, e.FinishReason = je.Request, je.FinishReason usage := je.Usage e.Usage = &usage if je.Message != nil { diff --git a/ui/src/archie/ActionTrailView.vue b/ui/src/archie/ActionTrailView.vue index 520d736be..52558e020 100644 --- a/ui/src/archie/ActionTrailView.vue +++ b/ui/src/archie/ActionTrailView.vue @@ -52,6 +52,26 @@ const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary"); {{ json(decision.action.input) }} +
+ Connector calls +
+
+ {{ c.connector }} {{ c.method }} + Request + + {{ json(c.request) }} + + + {{ c.error }} +
+
+
+
  • diff --git a/ui/src/archie/ComposerView.vue b/ui/src/archie/ComposerView.vue index 33d25a45c..c19ca5028 100644 --- a/ui/src/archie/ComposerView.vue +++ b/ui/src/archie/ComposerView.vue @@ -3,6 +3,7 @@ import { computed, onBeforeUnmount, onMounted, ref } from "vue"; import { RouterLink } from "vue-router"; import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty"; +import { Badge } from "@/components/ui/badge"; import { ReadError } from "@/components/ui/read-error"; import { Button } from "@/components/ui/button"; import { Section } from "@/components/ui/section"; @@ -109,6 +110,20 @@ onBeforeUnmount(() => controller.abort());

    {{ m.body }}

    +
    + Delivery attempts +
      +
    • + {{ a.channel }} + Attempt {{ a.attempt }} + {{ a.ok ? "Delivered" : "Failed" }} + {{ a.duration_ms }} ms + + {{ a.response }} + {{ a.error }} +
    • +
    +
    diff --git a/ui/src/archie/RunView.vue b/ui/src/archie/RunView.vue index 972e188ca..d980b6cbd 100644 --- a/ui/src/archie/RunView.vue +++ b/ui/src/archie/RunView.vue @@ -19,7 +19,7 @@ import { TraceTitle, } from "@/components/ui/trace"; import ActionTrailView from "./ActionTrailView.vue"; -import { runTrail, type Trail, type TrailEntry, Unauthorized } from "./api"; +import { runTrail, type Trail, type TrailEntry, type Usage, Unauthorized } from "./api"; import { runTone } from "./state"; /** A run's decision trail: what started it, every step in order, and how it ended. */ @@ -78,6 +78,12 @@ function title(e: TrailEntry): string { } const json = (v: unknown) => JSON.stringify(v, null, 2); + +const usageRows = (u: Usage) => [ + { label: "Prompt tokens", value: u.prompt_tokens.toLocaleString() }, + { label: "Completion tokens", value: u.completion_tokens.toLocaleString() }, + { label: "Total tokens", value: u.total_tokens.toLocaleString() }, +];