Files
sam 98c0c11cbb feat(chat): track session taint and hold mutations in tainted conversations (#2068)
A conversation gets a persisted tainted flag (migration 00040), set the
first time output from a tool whose declaration trust level is not
"trusted" enters the turn, before that output reaches the model. The
flag is only ever set, never cleared, so later turns and resumed
processes inherit it. In a tainted conversation every non-read-only
action is raised to a floor of propose_and_wait, so the existing hold
path renders the approval card with the exact judged action; read-only
lookups still run, and a hard-rule refusal stays a refusal. An
unreadable taint state or a failed taint write fails closed. Trust
levels live on connectors.Declaration; unknown levels are untrusted.
2026-11-21 03:15:04 +11:00

234 lines
10 KiB
Go

package chat
import (
"context"
"encoding/json/jsontext"
"errors"
"fmt"
"log/slog"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/chat/internal/store"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
)
// trustBox declares tools with explicit trust levels.
type trustBox struct{ decls []connectors.Declaration }
func (b trustBox) Declarations() []connectors.Declaration { return b.decls }
func (trustBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
// taintJudge decides at a fixed level and mirrors the real Execute's hold and
// refusal switches, recording the level it was last executed with.
type taintJudge struct {
Judge
level judgement.Level
executed judgement.Level
}
func (j *taintJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
return judgement.Decision{Key: req.Key, RunID: req.RunID, Level: j.level}, nil
}
func (j *taintJudge) Execute(_ context.Context, d judgement.Decision, _ jsontext.Value, _ string) (string, error) {
j.executed = d.Level
switch {
case d.Level == judgement.Never:
return "", fmt.Errorf("%w: hard rule", judgement.ErrRefused)
case !d.Level.Runs():
return "", fmt.Errorf("%w at %s", judgement.ErrHeld, d.Level)
}
return "ran", nil
}
// fakeApprover records the asks and answers them with a ready request.
type fakeApprover struct{ Approver }
func (fakeApprover) Ask(_ context.Context, _ uuid.UUID, key string, _ judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
return approvals.Request{ID: uuid.New(), Key: key, Summary: "approve this", StepDeadline: time.Now().Add(time.Hour)}, nil
}
// fakeSettings gives every held action a one-step escalation path.
type fakeSettings struct{ Settings }
func (fakeSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: alice}}, nil
}
// taintDeps builds the module dependencies around a store holding one
// conversation alice owns.
func taintDeps(t *testing.T, fs *fakeStore, box Toolbox, judge *taintJudge) (*Module, Deps, uuid.UUID) {
t.Helper()
session := uuid.New()
fs.sessions[session] = store.Session{ID: session, Owner: alice}
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
d := Deps{
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
Settings: fakeSettings{}, Toolbox: box, Judge: judge, Approver: fakeApprover{}, Memory: fakeMemory{},
}
m := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
return m, d, session
}
func untrustedLookup() connectors.Declaration {
return connectors.Declaration{Connector: "web", Kind: connectors.KindLookup, Name: "fetch", Schema: jsontext.Value(`{"type":"object"}`)}
}
func untrustedAction() connectors.Declaration {
return connectors.Declaration{Connector: "mail", Kind: connectors.KindAction, Name: "send", Schema: jsontext.Value(`{"type":"object"}`)}
}
// TestUntrustedOutputTaintsTheSession guards requirement 1 and 2: output from
// a tool whose trust level is not "trusted" sets the session's persisted
// flag before the output reaches the model, and nothing clears it afterwards,
// including a fresh module over the same store (a restart).
func TestUntrustedOutputTaintsTheSession(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{
untrustedLookup(),
{Connector: "internal", Kind: connectors.KindLookup, Name: "runs", TrustLevel: connectors.TrustTrusted, Schema: jsontext.Value(`{"type":"object"}`)},
}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); tainted {
t.Fatal("a fresh conversation is tainted before any tool ran")
}
out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`))
if err != nil || out != "someone" {
t.Fatalf("untrusted lookup = %q, %v; want its output", out, err)
}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("untrusted tool output did not set the session's taint flag")
}
// Trusted output later in the same thread does not clear the flag, and a
// new module over the same store (a restart) still sees it.
trusted := box.decls[1]
if _, err := m.call(ctx, d, turn, trusted, jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
fresh := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
_ = fresh
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("a trusted tool cleared the taint flag")
}
}
// TestUnknownTrustLevelIsUntrusted guards requirement 2: a declaration whose
// trust level is not recognised taints the session like an untrusted one.
func TestUnknownTrustLevelIsUntrusted(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
decl := untrustedLookup()
decl.TrustLevel = "mystery"
m, d, session := taintDeps(t, fs, trustBox{decls: []connectors.Declaration{decl}}, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
if _, err := m.call(ctx, d, store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}, decl, jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("an unknown trust level did not taint the session")
}
}
// TestTaintedSessionHoldsMutationsButNotReads guards requirement 3: once
// tainted, a non-read-only action is held for confirmation with the resolved
// action shown (the judge sees a floor of propose_and_wait), while a read-only
// lookup still runs.
func TestTaintedSessionHoldsMutationsButNotReads(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
// Taint the session with untrusted content.
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
// A mutation is held, not run: the judge is handed propose_and_wait, the
// tool never runs, and a waiting action is stored for the approval card.
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{"to":"x"}`))
if err != nil {
t.Fatalf("held action returned an error: %v", err)
}
if !strings.Contains(out, "held") {
t.Fatalf("tainted mutation = %q, want it held for approval", out)
}
if judge.executed != judgement.ProposeAndWait {
t.Fatalf("judge executed at %s, want %s", judge.executed, judgement.ProposeAndWait)
}
if len(fs.actions) != 1 {
t.Fatalf("waiting actions = %d, want 1 stored for the approval card", len(fs.actions))
}
// A read-only lookup still runs in the tainted session.
if out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil || out != "someone" {
t.Fatalf("tainted read-only lookup = %q, %v; want it to run", out, err)
}
}
// TestHardRefusalStaysARefusalWhenTainted guards requirement 3's other half: a
// hard-rule refusal (never) is not converted into a held action by taint.
func TestHardRefusalStaysARefusalWhenTainted(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.Never}
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
_, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
if !errors.Is(err, judgement.ErrRefused) {
t.Fatalf("tainted refusal = %v, want ErrRefused (not held)", err)
}
if len(fs.actions) != 0 {
t.Fatal("a refused action was stored as waiting")
}
}
// TestUnreadableTaintFailsClosed guards requirement 7: when the session's
// taint state cannot be read, the turn treats the session as tainted and holds
// the mutation.
func TestUnreadableTaintFailsClosed(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
m.store = &brokenTaintStore{fakeStore: fs}
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
if err != nil {
t.Fatalf("call = %v", err)
}
if !strings.Contains(out, "held") || judge.executed != judgement.ProposeAndWait {
t.Fatalf("unreadable taint: out %q executed %s, want held at propose_and_wait", out, judge.executed)
}
}
// brokenTaintStore fails to read the taint flag.
type brokenTaintStore struct{ *fakeStore }
func (b *brokenTaintStore) SessionTainted(context.Context, string, string, uuid.UUID) (bool, error) {
return false, errors.New("store unavailable")
}