Merge branch 'worktree-agent-a1bd90c329b751a72'
This commit is contained in:
15 files changed
+609
-44
No files matched your search
@@ -0,0 +1,142 @@
|
||||
// Package auditjson stores the exact request and response of a call for
|
||||
// audit: the message as JSON, with credentials removed and an oversized body
|
||||
// capped rather than dropped silently.
|
||||
package auditjson
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"google.golang.org/protobuf/encoding/protojson"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// MaxBytes is the most of one message that is stored. A larger message keeps
|
||||
// its first MaxBytes, with its full length and SHA-256.
|
||||
const MaxBytes = 1 << 20
|
||||
|
||||
// redacted replaces a credential.
|
||||
const redacted = "[redacted]"
|
||||
|
||||
// Payload is one stored message. JSON is the message itself when it fits under
|
||||
// MaxBytes. A message that doesn't fit keeps its first MaxBytes as Text, and
|
||||
// Length and SHA256 describe the whole of it.
|
||||
type Payload struct {
|
||||
JSON jsontext.Value `json:"json,omitzero"`
|
||||
Text string `json:"text,omitzero"`
|
||||
Truncated bool `json:"truncated,omitzero"`
|
||||
Length int64 `json:"length,omitzero"`
|
||||
SHA256 string `json:"sha256,omitzero"`
|
||||
}
|
||||
|
||||
// FromProto renders a protobuf message as protojson, with credentials removed
|
||||
// and the result capped. The message is what core sent or received on the
|
||||
// wire, never anything a connector resolved internally.
|
||||
func FromProto(msg proto.Message) Payload {
|
||||
raw, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(msg)
|
||||
if err != nil {
|
||||
return Payload{Text: "unrenderable: " + err.Error()}
|
||||
}
|
||||
return FromJSON(raw)
|
||||
}
|
||||
|
||||
// FromValue encodes v as JSON, with credentials removed and the result capped.
|
||||
func FromValue(v any) Payload {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return Payload{Text: "unrenderable: " + err.Error()}
|
||||
}
|
||||
return FromJSON(raw)
|
||||
}
|
||||
|
||||
// FromJSON redacts credentials in raw and caps the result.
|
||||
func FromJSON(raw jsontext.Value) Payload {
|
||||
clean := Redact(raw)
|
||||
if len(clean) <= MaxBytes {
|
||||
return Payload{JSON: clean}
|
||||
}
|
||||
sum := sha256.Sum256(clean)
|
||||
kept := MaxBytes
|
||||
for kept > 0 && !utf8.RuneStart(clean[kept]) {
|
||||
kept--
|
||||
}
|
||||
return Payload{
|
||||
Text: string(clean[:kept]), Truncated: true,
|
||||
Length: int64(len(clean)), SHA256: hex.EncodeToString(sum[:]),
|
||||
}
|
||||
}
|
||||
|
||||
// Redact replaces every credential in a JSON value with a marker. A key that
|
||||
// names a credential is redacted whole; a value that reads as an authorization
|
||||
// header is redacted wherever it appears. A payload that can't be decoded is
|
||||
// replaced rather than stored unchanged, because its credentials can't be
|
||||
// found.
|
||||
func Redact(raw jsontext.Value) jsontext.Value {
|
||||
var v any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return jsontext.Value(`"[unreadable payload]"`)
|
||||
}
|
||||
redactValue(v)
|
||||
out, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return jsontext.Value(`"[unreadable payload]"`)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactValue(v any) {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
for key, value := range x {
|
||||
if sensitiveKey(key) {
|
||||
x[key] = redacted
|
||||
continue
|
||||
}
|
||||
if s, ok := value.(string); ok && looksSecret(s) {
|
||||
x[key] = redacted
|
||||
continue
|
||||
}
|
||||
redactValue(value)
|
||||
}
|
||||
case []any:
|
||||
for _, value := range x {
|
||||
redactValue(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sensitiveKey reports whether a JSON key names a credential. The match is
|
||||
// deliberately broad: an audit that keeps a credential is worse than one that
|
||||
// hides a harmless field.
|
||||
func sensitiveKey(key string) bool {
|
||||
k := strings.ToLower(key)
|
||||
switch {
|
||||
case strings.Contains(k, "authorization"),
|
||||
strings.Contains(k, "cookie"),
|
||||
strings.Contains(k, "password"),
|
||||
strings.Contains(k, "passwd"),
|
||||
strings.Contains(k, "secret"),
|
||||
strings.Contains(k, "privatekey"),
|
||||
strings.Contains(k, "private_key"),
|
||||
strings.Contains(k, "credential"),
|
||||
strings.Contains(k, "apikey"),
|
||||
strings.Contains(k, "api_key"),
|
||||
strings.Contains(k, "api-key"):
|
||||
return true
|
||||
}
|
||||
compact := strings.NewReplacer("_", "", "-", "").Replace(k)
|
||||
return compact == "token" || strings.HasSuffix(compact, "token")
|
||||
}
|
||||
|
||||
// looksSecret reports whether a value is an authorization header, whatever
|
||||
// key it arrived under.
|
||||
func looksSecret(value string) bool {
|
||||
s := strings.TrimSpace(value)
|
||||
lower := strings.ToLower(s)
|
||||
return strings.HasPrefix(lower, "bearer ") || strings.HasPrefix(lower, "basic ") ||
|
||||
strings.Contains(s, "-----BEGIN ")
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package auditjson
|
||||
|
||||
import (
|
||||
"encoding/json/jsontext"
|
||||
json "encoding/json/v2"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestStoredPayloadKeepsCredentialsOut guards the audit rule that a stored
|
||||
// request or response never carries a credential, wherever it sits in the
|
||||
// message and however the key is spelled.
|
||||
func TestStoredPayloadKeepsCredentialsOut(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
secret string
|
||||
// kept is text that must survive redaction, so a payload isn't
|
||||
// blanked wholesale.
|
||||
kept string
|
||||
}{
|
||||
{
|
||||
name: "authorization header",
|
||||
in: `{"headers":{"Authorization":"Bearer sk-live-123","Content-Type":"application/json"},"body":"ok"}`,
|
||||
secret: "sk-live-123",
|
||||
kept: "application/json",
|
||||
},
|
||||
{
|
||||
name: "api key field",
|
||||
in: `{"input":{"api_key":"key-abc","repository":"archie"}}`,
|
||||
secret: "key-abc",
|
||||
kept: "archie",
|
||||
},
|
||||
{
|
||||
name: "nested password",
|
||||
in: `{"request":{"password":"hunter2"},"target":"db"}`,
|
||||
secret: "hunter2",
|
||||
kept: "db",
|
||||
},
|
||||
{
|
||||
name: "bearer value under an innocuous key",
|
||||
in: `{"headers":{"X-Custom":"Bearer opaque-456"},"event":"push"}`,
|
||||
secret: "opaque-456",
|
||||
kept: "push",
|
||||
},
|
||||
{
|
||||
name: "token field",
|
||||
in: `{"access_token":"tok-789","scope":"read"}`,
|
||||
secret: "tok-789",
|
||||
kept: "read",
|
||||
},
|
||||
{
|
||||
name: "camel case token field",
|
||||
in: `{"accessToken":"camel-123","scope":"write"}`,
|
||||
secret: "camel-123",
|
||||
kept: "write",
|
||||
},
|
||||
{
|
||||
name: "harmless fields survive",
|
||||
in: `{"prompt_tokens":5,"idempotency_key":"k1","total_tokens":9}`,
|
||||
secret: "",
|
||||
kept: "idempotency_key",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
p := FromJSON(jsontext.Value(tt.in))
|
||||
out, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal stored payload: %v", err)
|
||||
}
|
||||
if tt.secret != "" && strings.Contains(string(out), tt.secret) {
|
||||
t.Fatalf("stored payload contains the credential %q: %s", tt.secret, out)
|
||||
}
|
||||
if !strings.Contains(string(out), tt.kept) {
|
||||
t.Fatalf("stored payload dropped %q: %s", tt.kept, out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
-- +goose Up
|
||||
-- One row per delivery attempt, so a channel's response and its failures are
|
||||
-- kept rather than only the last error. Nothing is pruned: the volume is low
|
||||
-- and the record is what makes a delivery auditable.
|
||||
CREATE TABLE deliveries.attempts (
|
||||
message_seq bigint NOT NULL REFERENCES deliveries.messages (seq) ON DELETE CASCADE,
|
||||
attempt int NOT NULL,
|
||||
at timestamptz NOT NULL DEFAULT now(),
|
||||
ok boolean NOT NULL,
|
||||
response text NOT NULL DEFAULT '',
|
||||
error text NOT NULL DEFAULT '',
|
||||
duration_ms bigint NOT NULL,
|
||||
PRIMARY KEY (message_seq, attempt)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE deliveries.attempts;
|
||||
@@ -0,0 +1,54 @@
|
||||
package connectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/auditjson"
|
||||
)
|
||||
|
||||
// Call is one call core made to a connector or MCP server: the request it sent
|
||||
// and the response it received, rendered as JSON. Credentials are removed and
|
||||
// an oversized body is capped, so the record is safe to keep.
|
||||
type Call struct {
|
||||
Connector string `json:"connector"`
|
||||
Method string `json:"method"`
|
||||
Request auditjson.Payload `json:"request"`
|
||||
Response auditjson.Payload `json:"response,omitzero"`
|
||||
Error string `json:"error,omitzero"`
|
||||
}
|
||||
|
||||
// Recorder keeps each call core makes to a connector. The module that owns the
|
||||
// record supplies one on the context with WithRecorder; without one nothing is
|
||||
// recorded.
|
||||
type Recorder interface {
|
||||
RecordConnectorCall(ctx context.Context, call Call)
|
||||
}
|
||||
|
||||
type recorderKey struct{}
|
||||
|
||||
// WithRecorder makes every connector call made with ctx report to r.
|
||||
func WithRecorder(ctx context.Context, r Recorder) context.Context {
|
||||
return context.WithValue(ctx, recorderKey{}, r)
|
||||
}
|
||||
|
||||
// record hands c to the recorder on ctx, if there is one.
|
||||
func record(ctx context.Context, c Call) {
|
||||
if r, ok := ctx.Value(recorderKey{}).(Recorder); ok && r != nil {
|
||||
r.RecordConnectorCall(ctx, c)
|
||||
}
|
||||
}
|
||||
|
||||
// recordProto records one protobuf wire call. The response is rendered only
|
||||
// when the call returned one; a transport error is recorded instead.
|
||||
func recordProto(ctx context.Context, connector, method string, req, resp proto.Message, err error) {
|
||||
c := Call{Connector: connector, Method: method, Request: auditjson.FromProto(req)}
|
||||
switch {
|
||||
case err != nil:
|
||||
c.Error = err.Error()
|
||||
case resp != nil:
|
||||
c.Response = auditjson.FromProto(resp)
|
||||
}
|
||||
record(ctx, c)
|
||||
}
|
||||
@@ -215,7 +215,9 @@ func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Val
|
||||
if err := validate(schema, input); err != nil {
|
||||
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err)
|
||||
}
|
||||
resp, err := c.client.Deliver(ctx, &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key})
|
||||
req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}
|
||||
resp, err := c.client.Deliver(ctx, req)
|
||||
recordProto(ctx, c.name, "deliver", req, resp, err)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err)
|
||||
}
|
||||
@@ -264,9 +266,11 @@ func (c *Connector) TargetFacts(ctx context.Context, t Target) (TargetFacts, err
|
||||
if !c.facts {
|
||||
return TargetFacts{}, nil
|
||||
}
|
||||
resp, err := c.client.TargetFacts(ctx, &pb.TargetFactsRequest{Target: &pb.Target{
|
||||
req := &pb.TargetFactsRequest{Target: &pb.Target{
|
||||
Kind: t.Kind, Id: t.ID, Environment: t.Environment, Disposable: t.Disposable,
|
||||
}})
|
||||
}}
|
||||
resp, err := c.client.TargetFacts(ctx, req)
|
||||
recordProto(ctx, c.name, "target_facts", req, resp, err)
|
||||
if err != nil {
|
||||
return TargetFacts{}, fmt.Errorf("connector %q: target facts: %w", c.name, err)
|
||||
}
|
||||
@@ -302,7 +306,9 @@ func (c *Connector) Name() string { return c.name }
|
||||
// Parse turns a payload the caller has already verified into typed events,
|
||||
// each checked against its declared schema.
|
||||
func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]string) ([]Event, error) {
|
||||
resp, err := c.client.Parse(ctx, &pb.ParseRequest{Body: body, Headers: headers})
|
||||
req := &pb.ParseRequest{Body: body, Headers: headers}
|
||||
resp, err := c.client.Parse(ctx, req)
|
||||
recordProto(ctx, c.name, "parse", req, resp, err)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connector %q: parse: %w", c.name, err)
|
||||
}
|
||||
@@ -333,7 +339,9 @@ func (c *Connector) Resolve(ctx context.Context, name string, input jsontext.Val
|
||||
if _, err := c.action(name, input); err != nil {
|
||||
return Resolution{}, err
|
||||
}
|
||||
resp, err := c.client.Resolve(ctx, &pb.ResolveRequest{Action: name, Input: input})
|
||||
req := &pb.ResolveRequest{Action: name, Input: input}
|
||||
resp, err := c.client.Resolve(ctx, req)
|
||||
recordProto(ctx, c.name, "resolve", req, resp, err)
|
||||
if err != nil {
|
||||
return Resolution{}, fmt.Errorf("connector %q: resolve %q: %w", c.name, name, err)
|
||||
}
|
||||
@@ -357,9 +365,11 @@ func (c *Connector) Invoke(ctx context.Context, inv Invocation) (string, error)
|
||||
}
|
||||
inv.Input = Canonical(inv.Input)
|
||||
digest := Digest(inv.Action, inv.Input)
|
||||
resp, err := c.client.Invoke(ctx, &pb.InvokeRequest{
|
||||
req := &pb.InvokeRequest{
|
||||
Action: inv.Action, Input: inv.Input, IdempotencyKey: inv.IdempotencyKey, Digest: digest,
|
||||
})
|
||||
}
|
||||
resp, err := c.client.Invoke(ctx, req)
|
||||
recordProto(ctx, c.name, "invoke", req, resp, err)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("connector %q: invoke %q: %w", c.name, inv.Action, err)
|
||||
}
|
||||
@@ -382,7 +392,9 @@ func (c *Connector) Lookup(ctx context.Context, name string, input jsontext.Valu
|
||||
if err := validate(l.input, input); err != nil {
|
||||
return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
|
||||
}
|
||||
resp, err := c.client.Lookup(ctx, &pb.LookupRequest{Lookup: name, Input: input})
|
||||
req := &pb.LookupRequest{Lookup: name, Input: input}
|
||||
resp, err := c.client.Lookup(ctx, req)
|
||||
recordProto(ctx, c.name, "lookup", req, resp, err)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/google/jsonschema-go/jsonschema"
|
||||
sdk "github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/auditjson"
|
||||
"github.com/samcharles93/archie-core/modules/connectors/internal/egress"
|
||||
"github.com/samcharles93/archie-core/modules/connectors/internal/host"
|
||||
)
|
||||
@@ -182,16 +183,21 @@ func (s *Server) call(ctx context.Context, name, kind string, input jsontext.Val
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("mcp server %q: %w", s.name, err)
|
||||
}
|
||||
request := auditjson.FromValue(map[string]any{"tool": name, "arguments": args})
|
||||
ctx, cancel := context.WithTimeout(ctx, mcpCallTimeout)
|
||||
defer cancel()
|
||||
res, err := cs.CallTool(ctx, &sdk.CallToolParams{Name: name, Arguments: args})
|
||||
if err != nil {
|
||||
s.drop(cs, fmt.Errorf("tool %q: %w", name, err))
|
||||
record(ctx, Call{Connector: s.name, Method: kind, Request: request, Error: err.Error()})
|
||||
return "", false, fmt.Errorf("mcp server %q: tool %q: %w", s.name, name, err)
|
||||
}
|
||||
// A tool that answered with an error fails this call, not the run: the
|
||||
// message is the server's own, so a bad query reads as a bad query.
|
||||
out := capResult(render(res))
|
||||
record(ctx, Call{Connector: s.name, Method: kind, Request: request, Response: auditjson.FromValue(map[string]any{
|
||||
"output": out, "is_error": res.IsError,
|
||||
})})
|
||||
if res.IsError {
|
||||
return "", false, fmt.Errorf("mcp server %q: tool %q: %s", s.name, name, out)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,8 @@ type ApprovalRequest struct {
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// Message is a composer message as the API shows it.
|
||||
// Message is a composer message as the API shows it. Attempts are the delivery
|
||||
// attempts of the connector rows that carry the same message, one per channel.
|
||||
type Message struct {
|
||||
Seq int64 `json:"seq"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
@@ -83,6 +84,18 @@ type Message struct {
|
||||
Body string `json:"body"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitzero"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Attempts []Attempt `json:"attempts,omitzero"`
|
||||
}
|
||||
|
||||
// Attempt is one delivery attempt on one channel.
|
||||
type Attempt struct {
|
||||
Channel string `json:"channel"`
|
||||
Attempt int `json:"attempt"`
|
||||
At time.Time `json:"at"`
|
||||
OK bool `json:"ok"`
|
||||
Response string `json:"response,omitzero"`
|
||||
Error string `json:"error,omitzero"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
}
|
||||
|
||||
// Channels are the running connectors a message can go out on.
|
||||
@@ -196,15 +209,17 @@ func (m *Module) Work(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (m *Module) deliver(ctx context.Context, msg store.Message) {
|
||||
start := time.Now()
|
||||
external, err := m.sendTo(ctx, msg)
|
||||
took := time.Since(start)
|
||||
if err != nil {
|
||||
m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err)
|
||||
if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil {
|
||||
if err := m.store.Failed(ctx, msg.Seq, msg.Attempts, err.Error(), took); err != nil {
|
||||
m.log.ErrorContext(ctx, "record failed delivery", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := m.store.Sent(ctx, msg.Seq, external); err != nil {
|
||||
if err := m.store.Sent(ctx, msg.Seq, msg.Attempts, external, took); err != nil {
|
||||
m.log.ErrorContext(ctx, "record delivery", "error", err)
|
||||
}
|
||||
}
|
||||
@@ -263,9 +278,26 @@ func (m *Module) messages(w http.ResponseWriter, r *http.Request) {
|
||||
httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages")
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
keys = append(keys, row.Key)
|
||||
}
|
||||
attempts, err := m.store.AttemptsFor(r.Context(), m.orgID, keys)
|
||||
if err != nil {
|
||||
m.log.ErrorContext(r.Context(), "list delivery attempts", "error", err)
|
||||
httpjson.Error(w, http.StatusInternalServerError, "couldn't list messages")
|
||||
return
|
||||
}
|
||||
byKey := make(map[string][]Attempt, len(rows))
|
||||
for _, a := range attempts {
|
||||
byKey[a.Key] = append(byKey[a.Key], Attempt{
|
||||
Channel: a.Channel, Attempt: a.Attempt, At: a.At, OK: a.OK,
|
||||
Response: a.Response, Error: a.Error, DurationMS: a.DurationMS,
|
||||
})
|
||||
}
|
||||
out := make([]Message, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt}
|
||||
msg := Message{Seq: row.Seq, ID: row.ID, Kind: row.Kind, Title: row.Title, Body: row.Body, CreatedAt: row.CreatedAt, Attempts: byKey[row.Key]}
|
||||
if row.RunID.Valid {
|
||||
msg.RunID = &row.RunID.UUID
|
||||
}
|
||||
|
||||
@@ -126,14 +126,26 @@ func (s *Store) Claim(ctx context.Context, backoff func(attempts int) time.Durat
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
// Sent records that a channel confirmed a message.
|
||||
func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error {
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`,
|
||||
seq, externalID); err != nil {
|
||||
return fmt.Errorf("mark message sent: %w", err)
|
||||
}
|
||||
return nil
|
||||
// Attempt is one delivery attempt as the dashboard shows it.
|
||||
type Attempt struct {
|
||||
// Key is the message key the attempt's channel row shares with the
|
||||
// composer message.
|
||||
Key string `db:"key"`
|
||||
Channel string `db:"channel"`
|
||||
MessageSeq int64 `db:"message_seq"`
|
||||
Attempt int `db:"attempt"`
|
||||
At time.Time `db:"at"`
|
||||
OK bool `db:"ok"`
|
||||
Response string `db:"response"`
|
||||
Error string `db:"error"`
|
||||
DurationMS int64 `db:"duration_ms"`
|
||||
}
|
||||
|
||||
// Sent records that a channel confirmed a message, with the attempt that
|
||||
// confirmed it.
|
||||
func (s *Store) Sent(ctx context.Context, seq int64, attempt int, externalID string, took time.Duration) error {
|
||||
return s.attempt(ctx, seq, attempt, attemptOutcome{ok: true, response: externalID, took: took},
|
||||
`UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, externalID)
|
||||
}
|
||||
|
||||
// SentBy finds a message its channel confirmed under one of ids. The run is
|
||||
@@ -152,12 +164,58 @@ func (s *Store) SentBy(ctx context.Context, orgID, channel string, ids []string)
|
||||
return run, true, nil
|
||||
}
|
||||
|
||||
// Failed records why an attempt failed; the lease already set when it is
|
||||
// tried again.
|
||||
func (s *Store) Failed(ctx context.Context, seq int64, reason string) error {
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil {
|
||||
return fmt.Errorf("record failed delivery: %w", err)
|
||||
// Failed records why an attempt failed, with the attempt that failed; the
|
||||
// lease already set when it is tried again.
|
||||
func (s *Store) Failed(ctx context.Context, seq int64, attempt int, reason string, took time.Duration) error {
|
||||
return s.attempt(ctx, seq, attempt, attemptOutcome{failure: reason, took: took},
|
||||
`UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, reason)
|
||||
}
|
||||
|
||||
// attemptOutcome is one attempt's result.
|
||||
type attemptOutcome struct {
|
||||
ok bool
|
||||
response string
|
||||
failure string
|
||||
took time.Duration
|
||||
}
|
||||
|
||||
// attempt marks the message row and stores the attempt in one transaction, so
|
||||
// a retried message never loses the reason it was retried.
|
||||
func (s *Store) attempt(ctx context.Context, seq int64, attempt int, outcome attemptOutcome, mark, value string) error {
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
if _, err := tx.ExecContext(ctx, mark, seq, value); err != nil {
|
||||
return fmt.Errorf("record delivery: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO deliveries.attempts (message_seq, attempt, ok, response, error, duration_ms)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)`,
|
||||
seq, attempt, outcome.ok, outcome.response, outcome.failure, outcome.took.Milliseconds()); err != nil {
|
||||
return fmt.Errorf("record delivery attempt: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("record delivery attempt: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AttemptsFor returns the attempts of every connector delivery carrying one of
|
||||
// keys. Composer rows have none: they are stored already sent.
|
||||
func (s *Store) AttemptsFor(ctx context.Context, orgID string, keys []string) ([]Attempt, error) {
|
||||
if len(keys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var out []Attempt
|
||||
if err := s.db.SelectContext(ctx, &out, `
|
||||
SELECT m.key, m.channel, a.message_seq, a.attempt, a.at, a.ok, a.response, a.error, a.duration_ms
|
||||
FROM deliveries.attempts a
|
||||
JOIN deliveries.messages m ON m.seq = a.message_seq
|
||||
WHERE m.org_id = $1 AND m.key = ANY($2) AND m.channel <> 'composer'
|
||||
ORDER BY m.key, m.channel, a.attempt`, orgID, keys); err != nil {
|
||||
return nil, fmt.Errorf("list delivery attempts: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -60,6 +60,20 @@ func (s *Store) Record(ctx context.Context, d Decision) (Decision, error) {
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// AppendCall adds one connector call to a decision body's calls. The request
|
||||
// and response therefore sit with the judgement they justified, whether they
|
||||
// were part of judging the action or of running it.
|
||||
func (s *Store) AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error {
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
UPDATE judgement.decisions
|
||||
SET body = jsonb_set(body::jsonb, '{calls}',
|
||||
COALESCE(body::jsonb->'calls', '[]'::jsonb) || jsonb_build_array($3::jsonb))::json
|
||||
WHERE org_id = $1 AND id = $2`, orgID, id, call); err != nil {
|
||||
return fmt.Errorf("record connector call: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ForRun returns a run's decisions in the order they were made.
|
||||
func (s *Store) ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]Decision, error) {
|
||||
var out []Decision
|
||||
|
||||
@@ -157,6 +157,11 @@ type Decision struct {
|
||||
// Dry marks a decision made for a dry run. It is recorded for the run's
|
||||
// trail and counts toward nothing: not spend, not a class's history.
|
||||
Dry bool `json:"dry,omitzero"`
|
||||
// Calls are the calls to the connector behind this decision: the resolve
|
||||
// that placed its target, the target facts it was judged on, and the
|
||||
// invoke that ran it. Each holds the request core sent and the response it
|
||||
// received, with credentials removed.
|
||||
Calls []connectors.Call `json:"calls,omitzero"`
|
||||
}
|
||||
|
||||
// Owner is the id of the person who answers for the decision's target, or empty
|
||||
@@ -208,6 +213,29 @@ type FailureRecorder interface {
|
||||
type recorder interface {
|
||||
Record(ctx context.Context, d store.Decision) (store.Decision, error)
|
||||
ForRun(ctx context.Context, orgID string, runID uuid.UUID) ([]store.Decision, error)
|
||||
AppendCall(ctx context.Context, orgID string, id uuid.UUID, call string) error
|
||||
}
|
||||
|
||||
// callCollector keeps the connector calls one judgement made, so the decision
|
||||
// body records what core sent and received.
|
||||
type callCollector struct {
|
||||
mu sync.Mutex
|
||||
calls []connectors.Call
|
||||
}
|
||||
|
||||
// RecordConnectorCall keeps one call. Guards judge in parallel, so the
|
||||
// collector is safe for concurrent use.
|
||||
func (c *callCollector) RecordConnectorCall(_ context.Context, call connectors.Call) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.calls = append(c.calls, call)
|
||||
}
|
||||
|
||||
// take returns the calls collected so far.
|
||||
func (c *callCollector) take() []connectors.Call {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return slices.Clone(c.calls)
|
||||
}
|
||||
|
||||
// Module judges one organisation's actions.
|
||||
@@ -249,6 +277,8 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) {
|
||||
if !ok {
|
||||
return Decision{}, fmt.Errorf("%q isn't running", req.Connector)
|
||||
}
|
||||
calls := &callCollector{}
|
||||
ctx = connectors.WithRecorder(ctx, calls)
|
||||
res, err := tool.Resolve(ctx, req.Action, req.Input)
|
||||
if err != nil {
|
||||
return Decision{}, fmt.Errorf("resolve %s %s: %w", req.Connector, req.Action, err)
|
||||
@@ -259,6 +289,7 @@ func (m *Module) Assess(ctx context.Context, req Request) (Decision, error) {
|
||||
}
|
||||
d := m.decide(withRun(ctx, RunContext{RunID: req.RunID, Key: req.Key, Workflow: req.Workflow, Diagnosis: req.Diagnosis, Evidence: req.Evidence, Overlay: req.Overlay}), a)
|
||||
d.ID, d.RunID, d.Key, d.At, d.Dry = uuid.New(), req.RunID, req.Key, m.now().UTC(), req.Dry
|
||||
d.Calls = calls.take()
|
||||
return m.record(ctx, d)
|
||||
}
|
||||
|
||||
@@ -361,7 +392,10 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%q isn't running", d.Action.Connector)
|
||||
}
|
||||
calls := &callCollector{}
|
||||
ctx = connectors.WithRecorder(ctx, calls)
|
||||
out, err := tool.Invoke(ctx, connectors.Invocation{Action: d.Action.Name, Input: input, IdempotencyKey: key})
|
||||
m.keepCalls(ctx, d.ID, calls.take())
|
||||
// A shutdown cancels the call without the action having gone wrong, and a
|
||||
// dry run's failure is not evidence about the class.
|
||||
if err != nil && m.failed != nil && !d.Dry && !errors.Is(err, context.Canceled) {
|
||||
@@ -372,6 +406,25 @@ func (m *Module) invoke(ctx context.Context, d Decision, input jsontext.Value, k
|
||||
return out, err
|
||||
}
|
||||
|
||||
// keepCalls adds the connector calls one invoke made to its decision body, so
|
||||
// the executed action's request and response sit with the judgement that let
|
||||
// it run.
|
||||
func (m *Module) keepCalls(ctx context.Context, id uuid.UUID, calls []connectors.Call) {
|
||||
if id == uuid.Nil {
|
||||
return
|
||||
}
|
||||
for _, call := range calls {
|
||||
raw, err := json.Marshal(call)
|
||||
if err != nil {
|
||||
m.log.ErrorContext(ctx, "encode connector call", "decision", id, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := m.store.AppendCall(ctx, m.orgID, id, string(raw)); err != nil {
|
||||
m.log.ErrorContext(ctx, "record connector call", "decision", id, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Approval is a person's permission to run one decision's exact action. It
|
||||
// names the decision and the action's digest, so it covers that action and
|
||||
// nothing else, and it says who gave it and when.
|
||||
|
||||
+11
-5
@@ -101,11 +101,16 @@ type TrailEntry struct {
|
||||
Reasoning string `json:"reasoning,omitzero"`
|
||||
ToolCalls []ToolCallView `json:"tool_calls,omitzero"`
|
||||
Usage *chat.Usage `json:"usage,omitzero"`
|
||||
Input string `json:"input,omitzero"`
|
||||
Output string `json:"output,omitzero"`
|
||||
Value jsontext.Value `json:"value,omitzero"`
|
||||
Error string `json:"error,omitzero"`
|
||||
Unknown bool `json:"unknown,omitzero"`
|
||||
// Request is the model request exactly as it was sent: the messages, the
|
||||
// tools offered and the sampling settings.
|
||||
Request jsontext.Value `json:"request,omitzero"`
|
||||
// FinishReason is the wire finish reason of a model call.
|
||||
FinishReason string `json:"finish_reason,omitzero"`
|
||||
Input string `json:"input,omitzero"`
|
||||
Output string `json:"output,omitzero"`
|
||||
Value jsontext.Value `json:"value,omitzero"`
|
||||
Error string `json:"error,omitzero"`
|
||||
Unknown bool `json:"unknown,omitzero"`
|
||||
}
|
||||
|
||||
// ToolCallView is a tool call a model asked for.
|
||||
@@ -331,6 +336,7 @@ func entry(s store.Recorded) TrailEntry {
|
||||
if je.Kind == agentloop.EntryCompact {
|
||||
e.Kind = "compaction"
|
||||
}
|
||||
e.Request, e.FinishReason = je.Request, je.FinishReason
|
||||
usage := je.Usage
|
||||
e.Usage = &usage
|
||||
if je.Message != nil {
|
||||
|
||||
@@ -53,6 +53,26 @@ const ceilingTone = (v: Verdict): Tone => (v.failed ? "warn" : "secondary");
|
||||
<CodeBlockContent wrap class="max-h-40">{{ json(decision.action.input) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
|
||||
<details v-if="decision.calls?.length" class="text-caption">
|
||||
<summary class="cursor-pointer text-muted-foreground">Connector calls</summary>
|
||||
<div class="mt-1 flex flex-col gap-2">
|
||||
<div v-for="(c, i) in decision.calls" :key="i" class="flex flex-col gap-1 border-l pl-2">
|
||||
<span class="font-medium">{{ c.connector }} {{ c.method }}</span>
|
||||
<span class="text-muted-foreground">Request</span>
|
||||
<CodeBlock size="sm">
|
||||
<CodeBlockContent wrap class="max-h-40">{{ json(c.request) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<template v-if="c.response">
|
||||
<span class="text-muted-foreground">Response</span>
|
||||
<CodeBlock size="sm">
|
||||
<CodeBlockContent wrap class="max-h-40">{{ json(c.response) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</template>
|
||||
<span v-if="c.error" class="text-destructive">{{ c.error }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<ul class="flex flex-col gap-1.5">
|
||||
<li v-for="v in decision.verdicts" :key="v.guard" class="flex flex-col gap-0.5">
|
||||
<span class="flex items-center gap-2">
|
||||
|
||||
@@ -3,6 +3,7 @@ import { computed, onBeforeUnmount, onMounted, ref } from "vue";
|
||||
import { RouterLink } from "vue-router";
|
||||
|
||||
import { Empty, EmptyHeader, EmptyTitle } from "@/components/ui/empty";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { ReadError } from "@/components/ui/read-error";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Section } from "@/components/ui/section";
|
||||
@@ -109,6 +110,20 @@ onBeforeUnmount(() => controller.abort());
|
||||
</span>
|
||||
</template>
|
||||
<p class="whitespace-pre-wrap text-label">{{ m.body }}</p>
|
||||
<details v-if="m.attempts?.length" class="mt-2 text-caption">
|
||||
<summary class="cursor-pointer text-muted-foreground">Delivery attempts</summary>
|
||||
<ul class="mt-1 flex flex-col gap-1">
|
||||
<li v-for="a in m.attempts" :key="`${a.channel}:${a.attempt}`" class="flex flex-wrap items-center gap-2">
|
||||
<span class="font-medium">{{ a.channel }}</span>
|
||||
<span class="text-muted-foreground">Attempt {{ a.attempt }}</span>
|
||||
<Badge :variant="a.ok ? 'secondary' : 'danger'">{{ a.ok ? "Delivered" : "Failed" }}</Badge>
|
||||
<span class="tabular-nums text-muted-foreground">{{ a.duration_ms }} ms</span>
|
||||
<Timestamp :at="a.at" />
|
||||
<span v-if="a.response" class="font-mono">{{ a.response }}</span>
|
||||
<span v-if="a.error" class="text-destructive">{{ a.error }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
</details>
|
||||
<div v-if="wantsAnswer(m)" class="mt-3 flex items-center gap-2">
|
||||
<Button size="sm" :disabled="answering === m.request_id" @click="answer(m, true)">Approve</Button>
|
||||
<Button variant="outline" size="sm" :disabled="answering === m.request_id" @click="answer(m, false)">Deny</Button>
|
||||
|
||||
+44
-13
@@ -19,7 +19,7 @@ import {
|
||||
TraceTitle,
|
||||
} from "@/components/ui/trace";
|
||||
import ActionTrailView from "./ActionTrailView.vue";
|
||||
import { runTrail, type Trail, type TrailEntry, Unauthorized } from "./api";
|
||||
import { runTrail, type Trail, type TrailEntry, type Usage, Unauthorized } from "./api";
|
||||
import { runTone } from "./state";
|
||||
|
||||
/** A run's decision trail: what started it, every step in order, and how it ended. */
|
||||
@@ -78,6 +78,12 @@ function title(e: TrailEntry): string {
|
||||
}
|
||||
|
||||
const json = (v: unknown) => JSON.stringify(v, null, 2);
|
||||
|
||||
const usageRows = (u: Usage) => [
|
||||
{ label: "Prompt tokens", value: u.prompt_tokens.toLocaleString() },
|
||||
{ label: "Completion tokens", value: u.completion_tokens.toLocaleString() },
|
||||
{ label: "Total tokens", value: u.total_tokens.toLocaleString() },
|
||||
];
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -125,21 +131,46 @@ const json = (v: unknown) => JSON.stringify(v, null, 2);
|
||||
</TraceCollapsibleTrigger>
|
||||
<TraceCollapsibleContent>
|
||||
<div class="flex flex-col gap-2 text-label">
|
||||
<p v-if="e.reasoning" class="whitespace-pre-wrap text-muted-foreground">{{ e.reasoning }}</p>
|
||||
<p v-if="e.text" class="whitespace-pre-wrap">{{ e.text }}</p>
|
||||
<CodeBlock v-for="(c, i) in e.tool_calls" :key="i" size="sm">
|
||||
<CodeBlockContent wrap>{{ c.name }} {{ c.input }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<CodeBlock v-if="e.input" size="sm">
|
||||
<CodeBlockContent wrap>{{ e.input }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<CodeBlock v-if="e.output" size="sm">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ e.output }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<details v-if="e.request">
|
||||
<summary class="cursor-pointer text-caption text-muted-foreground">Request</summary>
|
||||
<CodeBlock size="sm" class="mt-1">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ json(e.request) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</details>
|
||||
<details v-if="e.kind === 'model' || e.kind === 'compaction'">
|
||||
<summary class="cursor-pointer text-caption text-muted-foreground">Response</summary>
|
||||
<div class="mt-1 flex flex-col gap-2">
|
||||
<p v-if="e.reasoning" class="whitespace-pre-wrap text-muted-foreground">{{ e.reasoning }}</p>
|
||||
<p v-if="e.text" class="whitespace-pre-wrap">{{ e.text }}</p>
|
||||
<CodeBlock v-for="(c, i) in e.tool_calls" :key="i" size="sm">
|
||||
<CodeBlockContent wrap>{{ c.name }} {{ c.input }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<p v-if="e.finish_reason" class="text-caption text-muted-foreground">
|
||||
Finish reason {{ e.finish_reason }}
|
||||
</p>
|
||||
<MetadataList v-if="e.usage" :rows="usageRows(e.usage)" />
|
||||
<p v-if="e.error" class="text-destructive">{{ e.error }}</p>
|
||||
</div>
|
||||
</details>
|
||||
<template v-else-if="e.kind === 'tool'">
|
||||
<details v-if="e.input">
|
||||
<summary class="cursor-pointer text-caption text-muted-foreground">Request</summary>
|
||||
<CodeBlock size="sm" class="mt-1">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ e.input }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</details>
|
||||
<details v-if="e.output || e.error">
|
||||
<summary class="cursor-pointer text-caption text-muted-foreground">Response</summary>
|
||||
<p v-if="e.error" class="mt-1 text-destructive">{{ e.error }}</p>
|
||||
<CodeBlock v-if="e.output" size="sm" class="mt-1">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ e.output }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
</details>
|
||||
</template>
|
||||
<CodeBlock v-if="e.kind === 'step' && e.value !== undefined" size="sm">
|
||||
<CodeBlockContent wrap class="max-h-64">{{ json(e.value) }}</CodeBlockContent>
|
||||
</CodeBlock>
|
||||
<p v-if="e.error" class="text-destructive">{{ e.error }}</p>
|
||||
<p v-if="e.error && e.kind === 'step'" class="text-destructive">{{ e.error }}</p>
|
||||
</div>
|
||||
</TraceCollapsibleContent>
|
||||
</TraceCollapsible>
|
||||
|
||||
@@ -32,6 +32,18 @@ export interface Message {
|
||||
body: string;
|
||||
expires_at?: string;
|
||||
created_at: string;
|
||||
/** Delivery attempts of the connector rows carrying this message, one per channel. */
|
||||
attempts?: MessageAttempt[];
|
||||
}
|
||||
|
||||
export interface MessageAttempt {
|
||||
channel: string;
|
||||
attempt: number;
|
||||
at: string;
|
||||
ok: boolean;
|
||||
response?: string;
|
||||
error?: string;
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export interface Usage {
|
||||
@@ -72,6 +84,9 @@ export interface TrailEntry {
|
||||
reasoning?: string;
|
||||
tool_calls?: { name: string; input: string }[];
|
||||
usage?: Usage;
|
||||
/** The model request exactly as it was sent: messages, tools offered and sampling settings. */
|
||||
request?: unknown;
|
||||
finish_reason?: string;
|
||||
input?: string;
|
||||
output?: string;
|
||||
value?: unknown;
|
||||
@@ -108,6 +123,14 @@ export interface Verdict {
|
||||
failed?: string;
|
||||
}
|
||||
|
||||
export interface ConnectorCall {
|
||||
connector: string;
|
||||
method: string;
|
||||
request: unknown;
|
||||
response?: unknown;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface Decision {
|
||||
id: string;
|
||||
key: string;
|
||||
@@ -126,6 +149,8 @@ export interface Decision {
|
||||
change_request_rule?: string;
|
||||
reason?: string;
|
||||
at: string;
|
||||
/** The connector calls behind the decision: resolve, target facts and invoke. */
|
||||
calls?: ConnectorCall[];
|
||||
}
|
||||
|
||||
export interface ApprovalRequest {
|
||||
|
||||
Reference in new issue
Block a user