diff --git a/internal/platform/pg/migrations/00040_chat_session_taint.sql b/internal/platform/pg/migrations/00040_chat_session_taint.sql new file mode 100644 index 000000000..8f64d3477 --- /dev/null +++ b/internal/platform/pg/migrations/00040_chat_session_taint.sql @@ -0,0 +1,8 @@ +-- +goose Up +-- A conversation that has ingested untrusted content is tainted for the rest +-- of its life: every later turn, including one a restarted process resumes, +-- tightens controls. The flag is only ever set, never cleared. +ALTER TABLE chat.sessions ADD COLUMN tainted boolean NOT NULL DEFAULT false; + +-- +goose Down +ALTER TABLE chat.sessions DROP COLUMN tainted; diff --git a/modules/chat/access_test.go b/modules/chat/access_test.go index 0f29f3c8f..ccea6278e 100644 --- a/modules/chat/access_test.go +++ b/modules/chat/access_test.go @@ -90,7 +90,7 @@ func TestTurnActsWithItsOwnersAccess(t *testing.T) { People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{}, } - m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} + m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()} action := connectors.Declaration{Connector: "pager", Kind: connectors.KindAction, Name: "page"} lookup := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"} cases := []struct { @@ -227,7 +227,7 @@ func TestOmittedToolCannotRun(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin)} box := staleBox{tool: &countingTool{}} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box} - m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} + m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()} ctx, err := m.actingAs(t.Context(), d, alice) if err != nil { t.Fatal(err) @@ -433,6 +433,37 @@ type fakeStore struct { sessions map[uuid.UUID]store.Session turns map[uuid.UUID]store.Turn events map[uuid.UUID][]store.Event + tainted map[uuid.UUID]bool + actions map[string]store.Action +} + +func (f *fakeStore) Taint(_ context.Context, _, _ string, id uuid.UUID) error { + if f.tainted == nil { + f.tainted = map[uuid.UUID]bool{} + } + f.tainted[id] = true + return nil +} + +func (f *fakeStore) SessionTainted(_ context.Context, _, _ string, id uuid.UUID) (bool, error) { + return f.tainted[id], nil +} + +func (f *fakeStore) PutAction(_ context.Context, _ string, a store.Action) (store.Action, error) { + if f.actions == nil { + f.actions = map[string]store.Action{} + } + f.actions[a.TurnID.String()+":"+a.Key] = a + return a, nil +} + +func (f *fakeStore) AddEvent(_ context.Context, turn uuid.UUID, kind string, data jsontext.Value) (int64, error) { + if f.events == nil { + f.events = map[uuid.UUID][]store.Event{} + } + seq := int64(len(f.events[turn]) + 1) + f.events[turn] = append(f.events[turn], store.Event{Seq: seq, TurnID: turn, Kind: kind, Data: data}) + return seq, nil } func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) { diff --git a/modules/chat/chat.go b/modules/chat/chat.go index a9cc959e8..75c0e3076 100644 --- a/modules/chat/chat.go +++ b/modules/chat/chat.go @@ -109,6 +109,8 @@ type sessionStore interface { Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error TitleIfUntitled(ctx context.Context, orgID string, id uuid.UUID, title string) error DeleteSession(ctx context.Context, orgID, owner string, id uuid.UUID) error + Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error + SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error) } type turnStore interface { diff --git a/modules/chat/internal/store/store.go b/modules/chat/internal/store/store.go index c6e1d5ca4..79b62b33a 100644 --- a/modules/chat/internal/store/store.go +++ b/modules/chat/internal/store/store.go @@ -22,6 +22,7 @@ type Session struct { ID uuid.UUID `db:"id"` Owner string `db:"owner_id"` Title string `db:"title"` + Tainted bool `db:"tainted"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` } @@ -81,7 +82,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} } const turnColumns = `seq, id, session_id, state, text, page, page_label, reply, reasoning, error, cancel, created_at, started_at, finished_at, (SELECT s.owner_id FROM chat.sessions s WHERE s.id = chat.turns.session_id) AS owner_id` -const sessionColumns = `id, owner_id, title, created_at, updated_at` +const sessionColumns = `id, owner_id, title, tainted, created_at, updated_at` // owned limits a turn query to the turns of owner's conversations in org. const owned = `session_id IN (SELECT s.id FROM chat.sessions s WHERE s.org_id = $1 AND s.owner_id = $2)` @@ -122,6 +123,28 @@ func (s *Store) Session(ctx context.Context, orgID, owner string, id uuid.UUID) return out, nil } +// Taint marks one of owner's conversations as having ingested untrusted +// content. It only ever sets the flag; nothing clears it. +func (s *Store) Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error { + res, err := s.db.ExecContext(ctx, ` + UPDATE chat.sessions SET tainted = true WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id) + return affected(res, err, "taint session") +} + +// SessionTainted reports whether one of owner's conversations is tainted. +func (s *Store) SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error) { + var tainted bool + err := s.db.GetContext(ctx, &tainted, ` + SELECT tainted FROM chat.sessions WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id) + if errors.Is(err, sql.ErrNoRows) { + return false, ErrNotFound + } + if err != nil { + return false, fmt.Errorf("read session taint: %w", err) + } + return tainted, nil +} + // Rename sets the title of one of owner's conversations. func (s *Store) Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error { res, err := s.db.ExecContext(ctx, ` diff --git a/modules/chat/taint_test.go b/modules/chat/taint_test.go new file mode 100644 index 000000000..9461eeb3a --- /dev/null +++ b/modules/chat/taint_test.go @@ -0,0 +1,233 @@ +package chat + +import ( + "context" + "encoding/json/jsontext" + "errors" + "fmt" + "log/slog" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/samcharles93/archie-core/internal/platform/hub" + "github.com/samcharles93/archie-core/modules/approvals" + "github.com/samcharles93/archie-core/modules/chat/internal/store" + "github.com/samcharles93/archie-core/modules/connectors" + "github.com/samcharles93/archie-core/modules/governance" + "github.com/samcharles93/archie-core/modules/judgement" +) + +// trustBox declares tools with explicit trust levels. +type trustBox struct{ decls []connectors.Declaration } + +func (b trustBox) Declarations() []connectors.Declaration { return b.decls } +func (trustBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true } + +// taintJudge decides at a fixed level and mirrors the real Execute's hold and +// refusal switches, recording the level it was last executed with. +type taintJudge struct { + Judge + level judgement.Level + executed judgement.Level +} + +func (j *taintJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) { + return judgement.Decision{Key: req.Key, RunID: req.RunID, Level: j.level}, nil +} + +func (j *taintJudge) Execute(_ context.Context, d judgement.Decision, _ jsontext.Value, _ string) (string, error) { + j.executed = d.Level + switch { + case d.Level == judgement.Never: + return "", fmt.Errorf("%w: hard rule", judgement.ErrRefused) + case !d.Level.Runs(): + return "", fmt.Errorf("%w at %s", judgement.ErrHeld, d.Level) + } + return "ran", nil +} + +// fakeApprover records the asks and answers them with a ready request. +type fakeApprover struct{ Approver } + +func (fakeApprover) Ask(_ context.Context, _ uuid.UUID, key string, _ judgement.Decision, _ []governance.PathStep) (approvals.Request, error) { + return approvals.Request{ID: uuid.New(), Key: key, Summary: "approve this", StepDeadline: time.Now().Add(time.Hour)}, nil +} + +// fakeSettings gives every held action a one-step escalation path. +type fakeSettings struct{ Settings } + +func (fakeSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) { + return []governance.PathStep{{Person: alice}}, nil +} + +// taintDeps builds the module dependencies around a store holding one +// conversation alice owns. +func taintDeps(t *testing.T, fs *fakeStore, box Toolbox, judge *taintJudge) (*Module, Deps, uuid.UUID) { + t.Helper() + session := uuid.New() + fs.sessions[session] = store.Session{ID: session, Owner: alice} + people := fakePeople{alice: person(alice, governance.RoleAdmin)} + d := Deps{ + People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), + Settings: fakeSettings{}, Toolbox: box, Judge: judge, Approver: fakeApprover{}, Memory: fakeMemory{}, + } + m := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()} + return m, d, session +} + +func untrustedLookup() connectors.Declaration { + return connectors.Declaration{Connector: "web", Kind: connectors.KindLookup, Name: "fetch", Schema: jsontext.Value(`{"type":"object"}`)} +} + +func untrustedAction() connectors.Declaration { + return connectors.Declaration{Connector: "mail", Kind: connectors.KindAction, Name: "send", Schema: jsontext.Value(`{"type":"object"}`)} +} + +// TestUntrustedOutputTaintsTheSession guards requirement 1 and 2: output from +// a tool whose trust level is not "trusted" sets the session's persisted +// flag before the output reaches the model, and nothing clears it afterwards, +// including a fresh module over the same store (a restart). +func TestUntrustedOutputTaintsTheSession(t *testing.T) { + fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}} + judge := &taintJudge{level: judgement.ActQuietly} + box := trustBox{decls: []connectors.Declaration{ + untrustedLookup(), + {Connector: "internal", Kind: connectors.KindLookup, Name: "runs", TrustLevel: connectors.TrustTrusted, Schema: jsontext.Value(`{"type":"object"}`)}, + }} + m, d, session := taintDeps(t, fs, box, judge) + ctx, err := m.actingAs(t.Context(), d, alice) + if err != nil { + t.Fatal(err) + } + turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice} + + if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); tainted { + t.Fatal("a fresh conversation is tainted before any tool ran") + } + out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)) + if err != nil || out != "someone" { + t.Fatalf("untrusted lookup = %q, %v; want its output", out, err) + } + if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted { + t.Fatal("untrusted tool output did not set the session's taint flag") + } + + // Trusted output later in the same thread does not clear the flag, and a + // new module over the same store (a restart) still sees it. + trusted := box.decls[1] + if _, err := m.call(ctx, d, turn, trusted, jsontext.Value(`{}`)); err != nil { + t.Fatal(err) + } + fresh := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()} + _ = fresh + if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted { + t.Fatal("a trusted tool cleared the taint flag") + } +} + +// TestUnknownTrustLevelIsUntrusted guards requirement 2: a declaration whose +// trust level is not recognised taints the session like an untrusted one. +func TestUnknownTrustLevelIsUntrusted(t *testing.T) { + fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}} + judge := &taintJudge{level: judgement.ActQuietly} + decl := untrustedLookup() + decl.TrustLevel = "mystery" + m, d, session := taintDeps(t, fs, trustBox{decls: []connectors.Declaration{decl}}, judge) + ctx, _ := m.actingAs(t.Context(), d, alice) + if _, err := m.call(ctx, d, store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}, decl, jsontext.Value(`{}`)); err != nil { + t.Fatal(err) + } + if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted { + t.Fatal("an unknown trust level did not taint the session") + } +} + +// TestTaintedSessionHoldsMutationsButNotReads guards requirement 3: once +// tainted, a non-read-only action is held for confirmation with the resolved +// action shown (the judge sees a floor of propose_and_wait), while a read-only +// lookup still runs. +func TestTaintedSessionHoldsMutationsButNotReads(t *testing.T) { + fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}} + judge := &taintJudge{level: judgement.ActQuietly} + box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}} + m, d, session := taintDeps(t, fs, box, judge) + ctx, _ := m.actingAs(t.Context(), d, alice) + turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice} + + // Taint the session with untrusted content. + if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil { + t.Fatal(err) + } + + // A mutation is held, not run: the judge is handed propose_and_wait, the + // tool never runs, and a waiting action is stored for the approval card. + out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{"to":"x"}`)) + if err != nil { + t.Fatalf("held action returned an error: %v", err) + } + if !strings.Contains(out, "held") { + t.Fatalf("tainted mutation = %q, want it held for approval", out) + } + if judge.executed != judgement.ProposeAndWait { + t.Fatalf("judge executed at %s, want %s", judge.executed, judgement.ProposeAndWait) + } + if len(fs.actions) != 1 { + t.Fatalf("waiting actions = %d, want 1 stored for the approval card", len(fs.actions)) + } + + // A read-only lookup still runs in the tainted session. + if out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil || out != "someone" { + t.Fatalf("tainted read-only lookup = %q, %v; want it to run", out, err) + } +} + +// TestHardRefusalStaysARefusalWhenTainted guards requirement 3's other half: a +// hard-rule refusal (never) is not converted into a held action by taint. +func TestHardRefusalStaysARefusalWhenTainted(t *testing.T) { + fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}} + judge := &taintJudge{level: judgement.Never} + box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}} + m, d, session := taintDeps(t, fs, box, judge) + ctx, _ := m.actingAs(t.Context(), d, alice) + turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice} + if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil { + t.Fatal(err) + } + _, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`)) + if !errors.Is(err, judgement.ErrRefused) { + t.Fatalf("tainted refusal = %v, want ErrRefused (not held)", err) + } + if len(fs.actions) != 0 { + t.Fatal("a refused action was stored as waiting") + } +} + +// TestUnreadableTaintFailsClosed guards requirement 7: when the session's +// taint state cannot be read, the turn treats the session as tainted and holds +// the mutation. +func TestUnreadableTaintFailsClosed(t *testing.T) { + fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}} + judge := &taintJudge{level: judgement.ActQuietly} + box := trustBox{decls: []connectors.Declaration{untrustedAction()}} + m, d, session := taintDeps(t, fs, box, judge) + m.store = &brokenTaintStore{fakeStore: fs} + ctx, _ := m.actingAs(t.Context(), d, alice) + turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice} + out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`)) + if err != nil { + t.Fatalf("call = %v", err) + } + if !strings.Contains(out, "held") || judge.executed != judgement.ProposeAndWait { + t.Fatalf("unreadable taint: out %q executed %s, want held at propose_and_wait", out, judge.executed) + } +} + +// brokenTaintStore fails to read the taint flag. +type brokenTaintStore struct{ *fakeStore } + +func (b *brokenTaintStore) SessionTainted(context.Context, string, string, uuid.UUID) (bool, error) { + return false, errors.New("store unavailable") +} diff --git a/modules/chat/tools.go b/modules/chat/tools.go index 4c8ad036b..f5086648a 100644 --- a/modules/chat/tools.go +++ b/modules/chat/tools.go @@ -173,23 +173,41 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors return "", errToolUnavailable } if decl.Kind == connectors.KindLookup { - out, found, err := tool.Lookup(ctx, decl.Name, input) - switch { - case err != nil: - return "", err - case !found: - return "not found", nil - } - return out, nil + return m.lookup(ctx, d, t, decl, tool, input) } - key := uuid.NewString() + return m.act(ctx, d, t, decl, input) +} + +// lookup runs a read-only tool and, when its output is untrusted, taints the +// session before that output reaches the model. +func (m *Module) lookup(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, tool connectors.Tool, input jsontext.Value) (string, error) { + out, found, err := tool.Lookup(ctx, decl.Name, input) + switch { + case err != nil: + return "", err + case !found: + return "not found", nil + } + if err := m.taintIfUntrusted(ctx, t, decl); err != nil { + return "", err + } + return out, nil +} + +// act judges an action and runs it only as decided. Taint tightens controls +// and never grants them: in a tainted conversation every non-read-only action +// waits for confirmation. +func (m *Module) act(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, input jsontext.Value) (string, error) { decision, err := d.Judge.Assess(ctx, judgement.Request{ - RunID: t.ID, Key: key, Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal, + RunID: t.ID, Key: uuid.NewString(), Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal, }) if err != nil { return "", err } - out, err := d.Judge.Execute(ctx, decision, input, key) + if m.sessionTainted(ctx, t) && !decl.ReadOnly { + decision = taintCeiling(decision) + } + out, err := d.Judge.Execute(ctx, decision, input, decision.Key) switch { case errors.Is(err, judgement.ErrHeld): return m.hold(ctx, d, t, decision) @@ -198,9 +216,48 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors case err != nil: return "", err } + if err := m.taintIfUntrusted(ctx, t, decl); err != nil { + return "", err + } return out, nil } +// sessionTainted reports whether the turn's conversation has ingested +// untrusted content. An unreadable state fails closed as tainted. +func (m *Module) sessionTainted(ctx context.Context, t store.Turn) bool { + tainted, err := m.store.SessionTainted(ctx, m.orgID, t.Owner, t.SessionID) + return err != nil || tainted +} + +// taintCeiling raises a runnable decision to propose-and-wait with a taint +// verdict, so the existing hold path renders the approval card. A refusal +// (never) is left alone: a confirmation never converts it into a permit. +func taintCeiling(d judgement.Decision) judgement.Decision { + if !d.Level.Runs() { + return d + } + d.Verdicts = append(d.Verdicts, judgement.Verdict{ + Guard: "taint", + Ceiling: judgement.ProposeAndWait, + Findings: []judgement.Finding{{Text: "this conversation has ingested untrusted content, so the action waits for confirmation"}}, + }) + d.Level = judgement.ProposeAndWait + return d +} + +// taintIfUntrusted sets the session's taint flag before a tool's output +// reaches the model, when the declaration's trust level is anything but +// trusted. A write that fails is an error, never silently clean output. +func (m *Module) taintIfUntrusted(ctx context.Context, t store.Turn, decl connectors.Declaration) error { + if decl.TrustLevel == connectors.TrustTrusted { + return nil + } + if err := m.store.Taint(ctx, m.orgID, t.Owner, t.SessionID); err != nil { + return fmt.Errorf("record the session's taint: %w", err) + } + return nil +} + // hold puts an action judgement held to the first step of its escalation // path and shows the request in the conversation. func (m *Module) hold(ctx context.Context, d Deps, t store.Turn, decision judgement.Decision) (string, error) { diff --git a/modules/connectors/connector.go b/modules/connectors/connector.go index f40f5c9c7..0c98486ce 100644 --- a/modules/connectors/connector.go +++ b/modules/connectors/connector.go @@ -103,8 +103,17 @@ type Declaration struct { Description string `json:"description"` Schema jsontext.Value `json:"schema"` ReadOnly bool `json:"read_only"` + // TrustLevel says whether the tool's output is trusted internal content. + // Anything but TrustTrusted, including empty, is treated as untrusted. + TrustLevel string `json:"trust_level,omitempty"` } +// Trust levels a declaration can carry. +const ( + TrustTrusted = "trusted" + TrustUntrusted = "untrusted" +) + // Declaration kinds. const ( KindLookup = "lookup"