feat(chat): track session taint and hold mutations in tainted conversations (#2068)

A conversation gets a persisted tainted flag (migration 00040), set the
first time output from a tool whose declaration trust level is not
"trusted" enters the turn, before that output reaches the model. The
flag is only ever set, never cleared, so later turns and resumed
processes inherit it. In a tainted conversation every non-read-only
action is raised to a floor of propose_and_wait, so the existing hold
path renders the approval card with the exact judged action; read-only
lookups still run, and a hard-rule refusal stays a refusal. An
unreadable taint state or a failed taint write fails closed. Trust
levels live on connectors.Declaration; unknown levels are untrusted.
This commit is contained in:
sam committed 2026-11-21 03:15:04 +11:00
1 parent 9af3b29342
commit 98c0c11cbb
7 files changed
+377 -14

No files matched your search

@@ -0,0 +1,8 @@
-- +goose Up
-- A conversation that has ingested untrusted content is tainted for the rest
-- of its life: every later turn, including one a restarted process resumes,
-- tightens controls. The flag is only ever set, never cleared.
ALTER TABLE chat.sessions ADD COLUMN tainted boolean NOT NULL DEFAULT false;
-- +goose Down
ALTER TABLE chat.sessions DROP COLUMN tainted;
+33 -2
View File
@@ -90,7 +90,7 @@ func TestTurnActsWithItsOwnersAccess(t *testing.T) {
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{},
}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
action := connectors.Declaration{Connector: "pager", Kind: connectors.KindAction, Name: "page"}
lookup := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
cases := []struct {
@@ -227,7 +227,7 @@ func TestOmittedToolCannotRun(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
box := staleBox{tool: &countingTool{}}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
@@ -433,6 +433,37 @@ type fakeStore struct {
sessions map[uuid.UUID]store.Session
turns map[uuid.UUID]store.Turn
events map[uuid.UUID][]store.Event
tainted map[uuid.UUID]bool
actions map[string]store.Action
}
func (f *fakeStore) Taint(_ context.Context, _, _ string, id uuid.UUID) error {
if f.tainted == nil {
f.tainted = map[uuid.UUID]bool{}
}
f.tainted[id] = true
return nil
}
func (f *fakeStore) SessionTainted(_ context.Context, _, _ string, id uuid.UUID) (bool, error) {
return f.tainted[id], nil
}
func (f *fakeStore) PutAction(_ context.Context, _ string, a store.Action) (store.Action, error) {
if f.actions == nil {
f.actions = map[string]store.Action{}
}
f.actions[a.TurnID.String()+":"+a.Key] = a
return a, nil
}
func (f *fakeStore) AddEvent(_ context.Context, turn uuid.UUID, kind string, data jsontext.Value) (int64, error) {
if f.events == nil {
f.events = map[uuid.UUID][]store.Event{}
}
seq := int64(len(f.events[turn]) + 1)
f.events[turn] = append(f.events[turn], store.Event{Seq: seq, TurnID: turn, Kind: kind, Data: data})
return seq, nil
}
func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) {
+2
View File
@@ -109,6 +109,8 @@ type sessionStore interface {
Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error
TitleIfUntitled(ctx context.Context, orgID string, id uuid.UUID, title string) error
DeleteSession(ctx context.Context, orgID, owner string, id uuid.UUID) error
Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error
SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error)
}
type turnStore interface {
+24 -1
View File
@@ -22,6 +22,7 @@ type Session struct {
ID uuid.UUID `db:"id"`
Owner string `db:"owner_id"`
Title string `db:"title"`
Tainted bool `db:"tainted"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
@@ -81,7 +82,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} }
const turnColumns = `seq, id, session_id, state, text, page, page_label, reply, reasoning, error, cancel, created_at, started_at, finished_at,
(SELECT s.owner_id FROM chat.sessions s WHERE s.id = chat.turns.session_id) AS owner_id`
const sessionColumns = `id, owner_id, title, created_at, updated_at`
const sessionColumns = `id, owner_id, title, tainted, created_at, updated_at`
// owned limits a turn query to the turns of owner's conversations in org.
const owned = `session_id IN (SELECT s.id FROM chat.sessions s WHERE s.org_id = $1 AND s.owner_id = $2)`
@@ -122,6 +123,28 @@ func (s *Store) Session(ctx context.Context, orgID, owner string, id uuid.UUID)
return out, nil
}
// Taint marks one of owner's conversations as having ingested untrusted
// content. It only ever sets the flag; nothing clears it.
func (s *Store) Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error {
res, err := s.db.ExecContext(ctx, `
UPDATE chat.sessions SET tainted = true WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id)
return affected(res, err, "taint session")
}
// SessionTainted reports whether one of owner's conversations is tainted.
func (s *Store) SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error) {
var tainted bool
err := s.db.GetContext(ctx, &tainted, `
SELECT tainted FROM chat.sessions WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id)
if errors.Is(err, sql.ErrNoRows) {
return false, ErrNotFound
}
if err != nil {
return false, fmt.Errorf("read session taint: %w", err)
}
return tainted, nil
}
// Rename sets the title of one of owner's conversations.
func (s *Store) Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error {
res, err := s.db.ExecContext(ctx, `
+233
View File
@@ -0,0 +1,233 @@
package chat
import (
"context"
"encoding/json/jsontext"
"errors"
"fmt"
"log/slog"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/approvals"
"github.com/samcharles93/archie-core/modules/chat/internal/store"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
)
// trustBox declares tools with explicit trust levels.
type trustBox struct{ decls []connectors.Declaration }
func (b trustBox) Declarations() []connectors.Declaration { return b.decls }
func (trustBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
// taintJudge decides at a fixed level and mirrors the real Execute's hold and
// refusal switches, recording the level it was last executed with.
type taintJudge struct {
Judge
level judgement.Level
executed judgement.Level
}
func (j *taintJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
return judgement.Decision{Key: req.Key, RunID: req.RunID, Level: j.level}, nil
}
func (j *taintJudge) Execute(_ context.Context, d judgement.Decision, _ jsontext.Value, _ string) (string, error) {
j.executed = d.Level
switch {
case d.Level == judgement.Never:
return "", fmt.Errorf("%w: hard rule", judgement.ErrRefused)
case !d.Level.Runs():
return "", fmt.Errorf("%w at %s", judgement.ErrHeld, d.Level)
}
return "ran", nil
}
// fakeApprover records the asks and answers them with a ready request.
type fakeApprover struct{ Approver }
func (fakeApprover) Ask(_ context.Context, _ uuid.UUID, key string, _ judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
return approvals.Request{ID: uuid.New(), Key: key, Summary: "approve this", StepDeadline: time.Now().Add(time.Hour)}, nil
}
// fakeSettings gives every held action a one-step escalation path.
type fakeSettings struct{ Settings }
func (fakeSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
return []governance.PathStep{{Person: alice}}, nil
}
// taintDeps builds the module dependencies around a store holding one
// conversation alice owns.
func taintDeps(t *testing.T, fs *fakeStore, box Toolbox, judge *taintJudge) (*Module, Deps, uuid.UUID) {
t.Helper()
session := uuid.New()
fs.sessions[session] = store.Session{ID: session, Owner: alice}
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
d := Deps{
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
Settings: fakeSettings{}, Toolbox: box, Judge: judge, Approver: fakeApprover{}, Memory: fakeMemory{},
}
m := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
return m, d, session
}
func untrustedLookup() connectors.Declaration {
return connectors.Declaration{Connector: "web", Kind: connectors.KindLookup, Name: "fetch", Schema: jsontext.Value(`{"type":"object"}`)}
}
func untrustedAction() connectors.Declaration {
return connectors.Declaration{Connector: "mail", Kind: connectors.KindAction, Name: "send", Schema: jsontext.Value(`{"type":"object"}`)}
}
// TestUntrustedOutputTaintsTheSession guards requirement 1 and 2: output from
// a tool whose trust level is not "trusted" sets the session's persisted
// flag before the output reaches the model, and nothing clears it afterwards,
// including a fresh module over the same store (a restart).
func TestUntrustedOutputTaintsTheSession(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{
untrustedLookup(),
{Connector: "internal", Kind: connectors.KindLookup, Name: "runs", TrustLevel: connectors.TrustTrusted, Schema: jsontext.Value(`{"type":"object"}`)},
}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); tainted {
t.Fatal("a fresh conversation is tainted before any tool ran")
}
out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`))
if err != nil || out != "someone" {
t.Fatalf("untrusted lookup = %q, %v; want its output", out, err)
}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("untrusted tool output did not set the session's taint flag")
}
// Trusted output later in the same thread does not clear the flag, and a
// new module over the same store (a restart) still sees it.
trusted := box.decls[1]
if _, err := m.call(ctx, d, turn, trusted, jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
fresh := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
_ = fresh
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("a trusted tool cleared the taint flag")
}
}
// TestUnknownTrustLevelIsUntrusted guards requirement 2: a declaration whose
// trust level is not recognised taints the session like an untrusted one.
func TestUnknownTrustLevelIsUntrusted(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
decl := untrustedLookup()
decl.TrustLevel = "mystery"
m, d, session := taintDeps(t, fs, trustBox{decls: []connectors.Declaration{decl}}, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
if _, err := m.call(ctx, d, store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}, decl, jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
t.Fatal("an unknown trust level did not taint the session")
}
}
// TestTaintedSessionHoldsMutationsButNotReads guards requirement 3: once
// tainted, a non-read-only action is held for confirmation with the resolved
// action shown (the judge sees a floor of propose_and_wait), while a read-only
// lookup still runs.
func TestTaintedSessionHoldsMutationsButNotReads(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
// Taint the session with untrusted content.
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
// A mutation is held, not run: the judge is handed propose_and_wait, the
// tool never runs, and a waiting action is stored for the approval card.
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{"to":"x"}`))
if err != nil {
t.Fatalf("held action returned an error: %v", err)
}
if !strings.Contains(out, "held") {
t.Fatalf("tainted mutation = %q, want it held for approval", out)
}
if judge.executed != judgement.ProposeAndWait {
t.Fatalf("judge executed at %s, want %s", judge.executed, judgement.ProposeAndWait)
}
if len(fs.actions) != 1 {
t.Fatalf("waiting actions = %d, want 1 stored for the approval card", len(fs.actions))
}
// A read-only lookup still runs in the tainted session.
if out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil || out != "someone" {
t.Fatalf("tainted read-only lookup = %q, %v; want it to run", out, err)
}
}
// TestHardRefusalStaysARefusalWhenTainted guards requirement 3's other half: a
// hard-rule refusal (never) is not converted into a held action by taint.
func TestHardRefusalStaysARefusalWhenTainted(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.Never}
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
t.Fatal(err)
}
_, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
if !errors.Is(err, judgement.ErrRefused) {
t.Fatalf("tainted refusal = %v, want ErrRefused (not held)", err)
}
if len(fs.actions) != 0 {
t.Fatal("a refused action was stored as waiting")
}
}
// TestUnreadableTaintFailsClosed guards requirement 7: when the session's
// taint state cannot be read, the turn treats the session as tainted and holds
// the mutation.
func TestUnreadableTaintFailsClosed(t *testing.T) {
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
judge := &taintJudge{level: judgement.ActQuietly}
box := trustBox{decls: []connectors.Declaration{untrustedAction()}}
m, d, session := taintDeps(t, fs, box, judge)
m.store = &brokenTaintStore{fakeStore: fs}
ctx, _ := m.actingAs(t.Context(), d, alice)
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
if err != nil {
t.Fatalf("call = %v", err)
}
if !strings.Contains(out, "held") || judge.executed != judgement.ProposeAndWait {
t.Fatalf("unreadable taint: out %q executed %s, want held at propose_and_wait", out, judge.executed)
}
}
// brokenTaintStore fails to read the taint flag.
type brokenTaintStore struct{ *fakeStore }
func (b *brokenTaintStore) SessionTainted(context.Context, string, string, uuid.UUID) (bool, error) {
return false, errors.New("store unavailable")
}
+68 -11
View File
@@ -173,23 +173,41 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors
return "", errToolUnavailable
}
if decl.Kind == connectors.KindLookup {
out, found, err := tool.Lookup(ctx, decl.Name, input)
switch {
case err != nil:
return "", err
case !found:
return "not found", nil
}
return out, nil
return m.lookup(ctx, d, t, decl, tool, input)
}
key := uuid.NewString()
return m.act(ctx, d, t, decl, input)
}
// lookup runs a read-only tool and, when its output is untrusted, taints the
// session before that output reaches the model.
func (m *Module) lookup(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, tool connectors.Tool, input jsontext.Value) (string, error) {
out, found, err := tool.Lookup(ctx, decl.Name, input)
switch {
case err != nil:
return "", err
case !found:
return "not found", nil
}
if err := m.taintIfUntrusted(ctx, t, decl); err != nil {
return "", err
}
return out, nil
}
// act judges an action and runs it only as decided. Taint tightens controls
// and never grants them: in a tainted conversation every non-read-only action
// waits for confirmation.
func (m *Module) act(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, input jsontext.Value) (string, error) {
decision, err := d.Judge.Assess(ctx, judgement.Request{
RunID: t.ID, Key: key, Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal,
RunID: t.ID, Key: uuid.NewString(), Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal,
})
if err != nil {
return "", err
}
out, err := d.Judge.Execute(ctx, decision, input, key)
if m.sessionTainted(ctx, t) && !decl.ReadOnly {
decision = taintCeiling(decision)
}
out, err := d.Judge.Execute(ctx, decision, input, decision.Key)
switch {
case errors.Is(err, judgement.ErrHeld):
return m.hold(ctx, d, t, decision)
@@ -198,9 +216,48 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors
case err != nil:
return "", err
}
if err := m.taintIfUntrusted(ctx, t, decl); err != nil {
return "", err
}
return out, nil
}
// sessionTainted reports whether the turn's conversation has ingested
// untrusted content. An unreadable state fails closed as tainted.
func (m *Module) sessionTainted(ctx context.Context, t store.Turn) bool {
tainted, err := m.store.SessionTainted(ctx, m.orgID, t.Owner, t.SessionID)
return err != nil || tainted
}
// taintCeiling raises a runnable decision to propose-and-wait with a taint
// verdict, so the existing hold path renders the approval card. A refusal
// (never) is left alone: a confirmation never converts it into a permit.
func taintCeiling(d judgement.Decision) judgement.Decision {
if !d.Level.Runs() {
return d
}
d.Verdicts = append(d.Verdicts, judgement.Verdict{
Guard: "taint",
Ceiling: judgement.ProposeAndWait,
Findings: []judgement.Finding{{Text: "this conversation has ingested untrusted content, so the action waits for confirmation"}},
})
d.Level = judgement.ProposeAndWait
return d
}
// taintIfUntrusted sets the session's taint flag before a tool's output
// reaches the model, when the declaration's trust level is anything but
// trusted. A write that fails is an error, never silently clean output.
func (m *Module) taintIfUntrusted(ctx context.Context, t store.Turn, decl connectors.Declaration) error {
if decl.TrustLevel == connectors.TrustTrusted {
return nil
}
if err := m.store.Taint(ctx, m.orgID, t.Owner, t.SessionID); err != nil {
return fmt.Errorf("record the session's taint: %w", err)
}
return nil
}
// hold puts an action judgement held to the first step of its escalation
// path and shows the request in the conversation.
func (m *Module) hold(ctx context.Context, d Deps, t store.Turn, decision judgement.Decision) (string, error) {
+9
View File
@@ -103,8 +103,17 @@ type Declaration struct {
Description string `json:"description"`
Schema jsontext.Value `json:"schema"`
ReadOnly bool `json:"read_only"`
// TrustLevel says whether the tool's output is trusted internal content.
// Anything but TrustTrusted, including empty, is treated as untrusted.
TrustLevel string `json:"trust_level,omitempty"`
}
// Trust levels a declaration can carry.
const (
TrustTrusted = "trusted"
TrustUntrusted = "untrusted"
)
// Declaration kinds.
const (
KindLookup = "lookup"