feat(chat): track session taint and hold mutations in tainted conversations (#2068)
A conversation gets a persisted tainted flag (migration 00040), set the first time output from a tool whose declaration trust level is not "trusted" enters the turn, before that output reaches the model. The flag is only ever set, never cleared, so later turns and resumed processes inherit it. In a tainted conversation every non-read-only action is raised to a floor of propose_and_wait, so the existing hold path renders the approval card with the exact judged action; read-only lookups still run, and a hard-rule refusal stays a refusal. An unreadable taint state or a failed taint write fails closed. Trust levels live on connectors.Declaration; unknown levels are untrusted.
This commit is contained in:
7 files changed
+377
-14
No files matched your search
@@ -0,0 +1,8 @@
|
||||
-- +goose Up
|
||||
-- A conversation that has ingested untrusted content is tainted for the rest
|
||||
-- of its life: every later turn, including one a restarted process resumes,
|
||||
-- tightens controls. The flag is only ever set, never cleared.
|
||||
ALTER TABLE chat.sessions ADD COLUMN tainted boolean NOT NULL DEFAULT false;
|
||||
|
||||
-- +goose Down
|
||||
ALTER TABLE chat.sessions DROP COLUMN tainted;
|
||||
@@ -90,7 +90,7 @@ func TestTurnActsWithItsOwnersAccess(t *testing.T) {
|
||||
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
|
||||
Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{},
|
||||
}
|
||||
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
action := connectors.Declaration{Connector: "pager", Kind: connectors.KindAction, Name: "page"}
|
||||
lookup := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
|
||||
cases := []struct {
|
||||
@@ -227,7 +227,7 @@ func TestOmittedToolCannotRun(t *testing.T) {
|
||||
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
|
||||
box := staleBox{tool: &countingTool{}}
|
||||
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
|
||||
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
ctx, err := m.actingAs(t.Context(), d, alice)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -433,6 +433,37 @@ type fakeStore struct {
|
||||
sessions map[uuid.UUID]store.Session
|
||||
turns map[uuid.UUID]store.Turn
|
||||
events map[uuid.UUID][]store.Event
|
||||
tainted map[uuid.UUID]bool
|
||||
actions map[string]store.Action
|
||||
}
|
||||
|
||||
func (f *fakeStore) Taint(_ context.Context, _, _ string, id uuid.UUID) error {
|
||||
if f.tainted == nil {
|
||||
f.tainted = map[uuid.UUID]bool{}
|
||||
}
|
||||
f.tainted[id] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) SessionTainted(_ context.Context, _, _ string, id uuid.UUID) (bool, error) {
|
||||
return f.tainted[id], nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) PutAction(_ context.Context, _ string, a store.Action) (store.Action, error) {
|
||||
if f.actions == nil {
|
||||
f.actions = map[string]store.Action{}
|
||||
}
|
||||
f.actions[a.TurnID.String()+":"+a.Key] = a
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) AddEvent(_ context.Context, turn uuid.UUID, kind string, data jsontext.Value) (int64, error) {
|
||||
if f.events == nil {
|
||||
f.events = map[uuid.UUID][]store.Event{}
|
||||
}
|
||||
seq := int64(len(f.events[turn]) + 1)
|
||||
f.events[turn] = append(f.events[turn], store.Event{Seq: seq, TurnID: turn, Kind: kind, Data: data})
|
||||
return seq, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) {
|
||||
|
||||
@@ -109,6 +109,8 @@ type sessionStore interface {
|
||||
Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error
|
||||
TitleIfUntitled(ctx context.Context, orgID string, id uuid.UUID, title string) error
|
||||
DeleteSession(ctx context.Context, orgID, owner string, id uuid.UUID) error
|
||||
Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error
|
||||
SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error)
|
||||
}
|
||||
|
||||
type turnStore interface {
|
||||
|
||||
@@ -22,6 +22,7 @@ type Session struct {
|
||||
ID uuid.UUID `db:"id"`
|
||||
Owner string `db:"owner_id"`
|
||||
Title string `db:"title"`
|
||||
Tainted bool `db:"tainted"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}
|
||||
@@ -81,7 +82,7 @@ func New(db *sqlx.DB) *Store { return &Store{db: db} }
|
||||
const turnColumns = `seq, id, session_id, state, text, page, page_label, reply, reasoning, error, cancel, created_at, started_at, finished_at,
|
||||
(SELECT s.owner_id FROM chat.sessions s WHERE s.id = chat.turns.session_id) AS owner_id`
|
||||
|
||||
const sessionColumns = `id, owner_id, title, created_at, updated_at`
|
||||
const sessionColumns = `id, owner_id, title, tainted, created_at, updated_at`
|
||||
|
||||
// owned limits a turn query to the turns of owner's conversations in org.
|
||||
const owned = `session_id IN (SELECT s.id FROM chat.sessions s WHERE s.org_id = $1 AND s.owner_id = $2)`
|
||||
@@ -122,6 +123,28 @@ func (s *Store) Session(ctx context.Context, orgID, owner string, id uuid.UUID)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Taint marks one of owner's conversations as having ingested untrusted
|
||||
// content. It only ever sets the flag; nothing clears it.
|
||||
func (s *Store) Taint(ctx context.Context, orgID, owner string, id uuid.UUID) error {
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
UPDATE chat.sessions SET tainted = true WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id)
|
||||
return affected(res, err, "taint session")
|
||||
}
|
||||
|
||||
// SessionTainted reports whether one of owner's conversations is tainted.
|
||||
func (s *Store) SessionTainted(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error) {
|
||||
var tainted bool
|
||||
err := s.db.GetContext(ctx, &tainted, `
|
||||
SELECT tainted FROM chat.sessions WHERE org_id = $1 AND owner_id = $2 AND id = $3`, orgID, owner, id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return false, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read session taint: %w", err)
|
||||
}
|
||||
return tainted, nil
|
||||
}
|
||||
|
||||
// Rename sets the title of one of owner's conversations.
|
||||
func (s *Store) Rename(ctx context.Context, orgID, owner string, id uuid.UUID, title string) error {
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
package chat
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json/jsontext"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/platform/hub"
|
||||
"github.com/samcharles93/archie-core/modules/approvals"
|
||||
"github.com/samcharles93/archie-core/modules/chat/internal/store"
|
||||
"github.com/samcharles93/archie-core/modules/connectors"
|
||||
"github.com/samcharles93/archie-core/modules/governance"
|
||||
"github.com/samcharles93/archie-core/modules/judgement"
|
||||
)
|
||||
|
||||
// trustBox declares tools with explicit trust levels.
|
||||
type trustBox struct{ decls []connectors.Declaration }
|
||||
|
||||
func (b trustBox) Declarations() []connectors.Declaration { return b.decls }
|
||||
func (trustBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
|
||||
|
||||
// taintJudge decides at a fixed level and mirrors the real Execute's hold and
|
||||
// refusal switches, recording the level it was last executed with.
|
||||
type taintJudge struct {
|
||||
Judge
|
||||
level judgement.Level
|
||||
executed judgement.Level
|
||||
}
|
||||
|
||||
func (j *taintJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
|
||||
return judgement.Decision{Key: req.Key, RunID: req.RunID, Level: j.level}, nil
|
||||
}
|
||||
|
||||
func (j *taintJudge) Execute(_ context.Context, d judgement.Decision, _ jsontext.Value, _ string) (string, error) {
|
||||
j.executed = d.Level
|
||||
switch {
|
||||
case d.Level == judgement.Never:
|
||||
return "", fmt.Errorf("%w: hard rule", judgement.ErrRefused)
|
||||
case !d.Level.Runs():
|
||||
return "", fmt.Errorf("%w at %s", judgement.ErrHeld, d.Level)
|
||||
}
|
||||
return "ran", nil
|
||||
}
|
||||
|
||||
// fakeApprover records the asks and answers them with a ready request.
|
||||
type fakeApprover struct{ Approver }
|
||||
|
||||
func (fakeApprover) Ask(_ context.Context, _ uuid.UUID, key string, _ judgement.Decision, _ []governance.PathStep) (approvals.Request, error) {
|
||||
return approvals.Request{ID: uuid.New(), Key: key, Summary: "approve this", StepDeadline: time.Now().Add(time.Hour)}, nil
|
||||
}
|
||||
|
||||
// fakeSettings gives every held action a one-step escalation path.
|
||||
type fakeSettings struct{ Settings }
|
||||
|
||||
func (fakeSettings) EscalationPath(context.Context, string, bool) ([]governance.PathStep, error) {
|
||||
return []governance.PathStep{{Person: alice}}, nil
|
||||
}
|
||||
|
||||
// taintDeps builds the module dependencies around a store holding one
|
||||
// conversation alice owns.
|
||||
func taintDeps(t *testing.T, fs *fakeStore, box Toolbox, judge *taintJudge) (*Module, Deps, uuid.UUID) {
|
||||
t.Helper()
|
||||
session := uuid.New()
|
||||
fs.sessions[session] = store.Session{ID: session, Owner: alice}
|
||||
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
|
||||
d := Deps{
|
||||
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
|
||||
Settings: fakeSettings{}, Toolbox: box, Judge: judge, Approver: fakeApprover{}, Memory: fakeMemory{},
|
||||
}
|
||||
m := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
return m, d, session
|
||||
}
|
||||
|
||||
func untrustedLookup() connectors.Declaration {
|
||||
return connectors.Declaration{Connector: "web", Kind: connectors.KindLookup, Name: "fetch", Schema: jsontext.Value(`{"type":"object"}`)}
|
||||
}
|
||||
|
||||
func untrustedAction() connectors.Declaration {
|
||||
return connectors.Declaration{Connector: "mail", Kind: connectors.KindAction, Name: "send", Schema: jsontext.Value(`{"type":"object"}`)}
|
||||
}
|
||||
|
||||
// TestUntrustedOutputTaintsTheSession guards requirement 1 and 2: output from
|
||||
// a tool whose trust level is not "trusted" sets the session's persisted
|
||||
// flag before the output reaches the model, and nothing clears it afterwards,
|
||||
// including a fresh module over the same store (a restart).
|
||||
func TestUntrustedOutputTaintsTheSession(t *testing.T) {
|
||||
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
|
||||
judge := &taintJudge{level: judgement.ActQuietly}
|
||||
box := trustBox{decls: []connectors.Declaration{
|
||||
untrustedLookup(),
|
||||
{Connector: "internal", Kind: connectors.KindLookup, Name: "runs", TrustLevel: connectors.TrustTrusted, Schema: jsontext.Value(`{"type":"object"}`)},
|
||||
}}
|
||||
m, d, session := taintDeps(t, fs, box, judge)
|
||||
ctx, err := m.actingAs(t.Context(), d, alice)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
|
||||
|
||||
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); tainted {
|
||||
t.Fatal("a fresh conversation is tainted before any tool ran")
|
||||
}
|
||||
out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`))
|
||||
if err != nil || out != "someone" {
|
||||
t.Fatalf("untrusted lookup = %q, %v; want its output", out, err)
|
||||
}
|
||||
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
|
||||
t.Fatal("untrusted tool output did not set the session's taint flag")
|
||||
}
|
||||
|
||||
// Trusted output later in the same thread does not clear the flag, and a
|
||||
// new module over the same store (a restart) still sees it.
|
||||
trusted := box.decls[1]
|
||||
if _, err := m.call(ctx, d, turn, trusted, jsontext.Value(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh := &Module{orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New()}
|
||||
_ = fresh
|
||||
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
|
||||
t.Fatal("a trusted tool cleared the taint flag")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownTrustLevelIsUntrusted guards requirement 2: a declaration whose
|
||||
// trust level is not recognised taints the session like an untrusted one.
|
||||
func TestUnknownTrustLevelIsUntrusted(t *testing.T) {
|
||||
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
|
||||
judge := &taintJudge{level: judgement.ActQuietly}
|
||||
decl := untrustedLookup()
|
||||
decl.TrustLevel = "mystery"
|
||||
m, d, session := taintDeps(t, fs, trustBox{decls: []connectors.Declaration{decl}}, judge)
|
||||
ctx, _ := m.actingAs(t.Context(), d, alice)
|
||||
if _, err := m.call(ctx, d, store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}, decl, jsontext.Value(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tainted, _ := fs.SessionTainted(t.Context(), "default", alice, session); !tainted {
|
||||
t.Fatal("an unknown trust level did not taint the session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTaintedSessionHoldsMutationsButNotReads guards requirement 3: once
|
||||
// tainted, a non-read-only action is held for confirmation with the resolved
|
||||
// action shown (the judge sees a floor of propose_and_wait), while a read-only
|
||||
// lookup still runs.
|
||||
func TestTaintedSessionHoldsMutationsButNotReads(t *testing.T) {
|
||||
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
|
||||
judge := &taintJudge{level: judgement.ActQuietly}
|
||||
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
|
||||
m, d, session := taintDeps(t, fs, box, judge)
|
||||
ctx, _ := m.actingAs(t.Context(), d, alice)
|
||||
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
|
||||
|
||||
// Taint the session with untrusted content.
|
||||
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A mutation is held, not run: the judge is handed propose_and_wait, the
|
||||
// tool never runs, and a waiting action is stored for the approval card.
|
||||
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{"to":"x"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("held action returned an error: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "held") {
|
||||
t.Fatalf("tainted mutation = %q, want it held for approval", out)
|
||||
}
|
||||
if judge.executed != judgement.ProposeAndWait {
|
||||
t.Fatalf("judge executed at %s, want %s", judge.executed, judgement.ProposeAndWait)
|
||||
}
|
||||
if len(fs.actions) != 1 {
|
||||
t.Fatalf("waiting actions = %d, want 1 stored for the approval card", len(fs.actions))
|
||||
}
|
||||
|
||||
// A read-only lookup still runs in the tainted session.
|
||||
if out, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil || out != "someone" {
|
||||
t.Fatalf("tainted read-only lookup = %q, %v; want it to run", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHardRefusalStaysARefusalWhenTainted guards requirement 3's other half: a
|
||||
// hard-rule refusal (never) is not converted into a held action by taint.
|
||||
func TestHardRefusalStaysARefusalWhenTainted(t *testing.T) {
|
||||
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
|
||||
judge := &taintJudge{level: judgement.Never}
|
||||
box := trustBox{decls: []connectors.Declaration{untrustedLookup(), untrustedAction()}}
|
||||
m, d, session := taintDeps(t, fs, box, judge)
|
||||
ctx, _ := m.actingAs(t.Context(), d, alice)
|
||||
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
|
||||
if _, err := m.call(ctx, d, turn, untrustedLookup(), jsontext.Value(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
|
||||
if !errors.Is(err, judgement.ErrRefused) {
|
||||
t.Fatalf("tainted refusal = %v, want ErrRefused (not held)", err)
|
||||
}
|
||||
if len(fs.actions) != 0 {
|
||||
t.Fatal("a refused action was stored as waiting")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnreadableTaintFailsClosed guards requirement 7: when the session's
|
||||
// taint state cannot be read, the turn treats the session as tainted and holds
|
||||
// the mutation.
|
||||
func TestUnreadableTaintFailsClosed(t *testing.T) {
|
||||
fs := &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}
|
||||
judge := &taintJudge{level: judgement.ActQuietly}
|
||||
box := trustBox{decls: []connectors.Declaration{untrustedAction()}}
|
||||
m, d, session := taintDeps(t, fs, box, judge)
|
||||
m.store = &brokenTaintStore{fakeStore: fs}
|
||||
ctx, _ := m.actingAs(t.Context(), d, alice)
|
||||
turn := store.Turn{ID: uuid.New(), SessionID: session, Owner: alice}
|
||||
out, err := m.call(ctx, d, turn, untrustedAction(), jsontext.Value(`{}`))
|
||||
if err != nil {
|
||||
t.Fatalf("call = %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "held") || judge.executed != judgement.ProposeAndWait {
|
||||
t.Fatalf("unreadable taint: out %q executed %s, want held at propose_and_wait", out, judge.executed)
|
||||
}
|
||||
}
|
||||
|
||||
// brokenTaintStore fails to read the taint flag.
|
||||
type brokenTaintStore struct{ *fakeStore }
|
||||
|
||||
func (b *brokenTaintStore) SessionTainted(context.Context, string, string, uuid.UUID) (bool, error) {
|
||||
return false, errors.New("store unavailable")
|
||||
}
|
||||
+68
-11
@@ -173,23 +173,41 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors
|
||||
return "", errToolUnavailable
|
||||
}
|
||||
if decl.Kind == connectors.KindLookup {
|
||||
out, found, err := tool.Lookup(ctx, decl.Name, input)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", err
|
||||
case !found:
|
||||
return "not found", nil
|
||||
}
|
||||
return out, nil
|
||||
return m.lookup(ctx, d, t, decl, tool, input)
|
||||
}
|
||||
key := uuid.NewString()
|
||||
return m.act(ctx, d, t, decl, input)
|
||||
}
|
||||
|
||||
// lookup runs a read-only tool and, when its output is untrusted, taints the
|
||||
// session before that output reaches the model.
|
||||
func (m *Module) lookup(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, tool connectors.Tool, input jsontext.Value) (string, error) {
|
||||
out, found, err := tool.Lookup(ctx, decl.Name, input)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", err
|
||||
case !found:
|
||||
return "not found", nil
|
||||
}
|
||||
if err := m.taintIfUntrusted(ctx, t, decl); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// act judges an action and runs it only as decided. Taint tightens controls
|
||||
// and never grants them: in a tainted conversation every non-read-only action
|
||||
// waits for confirmation.
|
||||
func (m *Module) act(ctx context.Context, d Deps, t store.Turn, decl connectors.Declaration, input jsontext.Value) (string, error) {
|
||||
decision, err := d.Judge.Assess(ctx, judgement.Request{
|
||||
RunID: t.ID, Key: key, Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal,
|
||||
RunID: t.ID, Key: uuid.NewString(), Connector: decl.Connector, Action: decl.Name, Input: input, Workflow: principal,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := d.Judge.Execute(ctx, decision, input, key)
|
||||
if m.sessionTainted(ctx, t) && !decl.ReadOnly {
|
||||
decision = taintCeiling(decision)
|
||||
}
|
||||
out, err := d.Judge.Execute(ctx, decision, input, decision.Key)
|
||||
switch {
|
||||
case errors.Is(err, judgement.ErrHeld):
|
||||
return m.hold(ctx, d, t, decision)
|
||||
@@ -198,9 +216,48 @@ func (m *Module) call(ctx context.Context, d Deps, t store.Turn, decl connectors
|
||||
case err != nil:
|
||||
return "", err
|
||||
}
|
||||
if err := m.taintIfUntrusted(ctx, t, decl); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sessionTainted reports whether the turn's conversation has ingested
|
||||
// untrusted content. An unreadable state fails closed as tainted.
|
||||
func (m *Module) sessionTainted(ctx context.Context, t store.Turn) bool {
|
||||
tainted, err := m.store.SessionTainted(ctx, m.orgID, t.Owner, t.SessionID)
|
||||
return err != nil || tainted
|
||||
}
|
||||
|
||||
// taintCeiling raises a runnable decision to propose-and-wait with a taint
|
||||
// verdict, so the existing hold path renders the approval card. A refusal
|
||||
// (never) is left alone: a confirmation never converts it into a permit.
|
||||
func taintCeiling(d judgement.Decision) judgement.Decision {
|
||||
if !d.Level.Runs() {
|
||||
return d
|
||||
}
|
||||
d.Verdicts = append(d.Verdicts, judgement.Verdict{
|
||||
Guard: "taint",
|
||||
Ceiling: judgement.ProposeAndWait,
|
||||
Findings: []judgement.Finding{{Text: "this conversation has ingested untrusted content, so the action waits for confirmation"}},
|
||||
})
|
||||
d.Level = judgement.ProposeAndWait
|
||||
return d
|
||||
}
|
||||
|
||||
// taintIfUntrusted sets the session's taint flag before a tool's output
|
||||
// reaches the model, when the declaration's trust level is anything but
|
||||
// trusted. A write that fails is an error, never silently clean output.
|
||||
func (m *Module) taintIfUntrusted(ctx context.Context, t store.Turn, decl connectors.Declaration) error {
|
||||
if decl.TrustLevel == connectors.TrustTrusted {
|
||||
return nil
|
||||
}
|
||||
if err := m.store.Taint(ctx, m.orgID, t.Owner, t.SessionID); err != nil {
|
||||
return fmt.Errorf("record the session's taint: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// hold puts an action judgement held to the first step of its escalation
|
||||
// path and shows the request in the conversation.
|
||||
func (m *Module) hold(ctx context.Context, d Deps, t store.Turn, decision judgement.Decision) (string, error) {
|
||||
|
||||
@@ -103,8 +103,17 @@ type Declaration struct {
|
||||
Description string `json:"description"`
|
||||
Schema jsontext.Value `json:"schema"`
|
||||
ReadOnly bool `json:"read_only"`
|
||||
// TrustLevel says whether the tool's output is trusted internal content.
|
||||
// Anything but TrustTrusted, including empty, is treated as untrusted.
|
||||
TrustLevel string `json:"trust_level,omitempty"`
|
||||
}
|
||||
|
||||
// Trust levels a declaration can carry.
|
||||
const (
|
||||
TrustTrusted = "trusted"
|
||||
TrustUntrusted = "untrusted"
|
||||
)
|
||||
|
||||
// Declaration kinds.
|
||||
const (
|
||||
KindLookup = "lookup"
|
||||
|
||||
Reference in new issue
Block a user