Files
archie/modules/chat/access_test.go
T
sam 96fb0afcd9 feat(chat): offer the model only the tools its person may use (#2067)
Discovery now applies the same per-tool policy decision as invocation: a connector lookup needs read, an action needs run, and each builtin carries the action and resource kind it checks. The model's tool schemas and the system prompt's summaries come from that one filtered set. A name that two declarations, or a declaration and a builtin, both claim is withheld from all of them.

The stream client fixes the wire tools for the whole turn, so a guarded provider re-reads the owner's access and the running declarations before every model request. If an offered tool is no longer permitted or declared, the turn stops before the request instead of describing a stale set. A call to a declaration the registry no longer holds is refused before anything runs, naming nothing.

Memory notes are left out of the prompt when the person may not read settings, so the tool filter is not bypassed by prompt context.
2026-11-21 01:45:03 +11:00

518 lines
18 KiB
Go

package chat
import (
"context"
"encoding/json/jsontext"
"errors"
"io"
"log/slog"
"maps"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/internal/platform/httpapi"
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/chat/internal/store"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/memory"
)
const (
alice = "alice"
bob = "bob"
)
// person is a principal holding role in the operating org.
func person(id string, role governance.Role) governance.Principal {
return governance.Principal{
IdentityID: governance.IdentityID(id), Org: governance.DefaultOrgID,
Memberships: []governance.Membership{{IdentityID: governance.IdentityID(id), OrgID: governance.DefaultOrgID, Role: role}},
}
}
// TestConversationsBelongToTheirOwner guards the chat API's ownership
// boundary: a person reads and continues their own conversations, and every
// chat route answers anyone else as if the conversation did not exist.
func TestConversationsBelongToTheirOwner(t *testing.T) {
session, turn := uuid.New(), uuid.New()
cases := []struct {
name, method, path, body string
ok int
}{
{"rename", http.MethodPatch, "/api/chat/sessions/" + session.String(), `{"title":"mine"}`, http.StatusOK},
{"list turns", http.MethodGet, "/api/chat/sessions/" + session.String() + "/turns", "", http.StatusOK},
{"send", http.MethodPost, "/api/chat/sessions/" + session.String() + "/turns", `{"text":"hello"}`, http.StatusAccepted},
{"get turn", http.MethodGet, "/api/chat/turns/" + turn.String(), "", http.StatusOK},
{"cancel", http.MethodPost, "/api/chat/turns/" + turn.String() + "/cancel", "", http.StatusOK},
{"stream", http.MethodGet, "/api/chat/turns/" + turn.String() + "/events", "", http.StatusOK},
{"delete", http.MethodDelete, "/api/chat/sessions/" + session.String(), "", http.StatusNoContent},
}
for _, tc := range cases {
for _, as := range []string{alice, bob} {
t.Run(tc.name+" as "+as, func(t *testing.T) {
srv := chatServer(t, session, turn)
want := tc.ok
if as != alice {
want = http.StatusNotFound
}
if got, _ := call(t, srv, as, tc.method, tc.path, tc.body); got != want {
t.Fatalf("%s %s as %s = %d, want %d", tc.method, tc.path, as, got, want)
}
})
}
}
t.Run("list sessions", func(t *testing.T) {
srv := chatServer(t, session, turn)
for as, sees := range map[string]bool{alice: true, bob: false} {
code, body := call(t, srv, as, http.MethodGet, "/api/chat/sessions", "")
if code != http.StatusOK || strings.Contains(body, session.String()) != sees {
t.Fatalf("list as %s = %d %s, want it to show the conversation: %v", as, code, body, sees)
}
}
})
}
// TestTurnActsWithItsOwnersAccess guards the tool boundary: the same tool
// call runs in an administrator's conversation and is refused in a viewer's,
// so the agent never reaches further than the person chatting.
func TestTurnActsWithItsOwnersAccess(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)}
d := Deps{
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{},
}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
action := connectors.Declaration{Connector: "pager", Kind: connectors.KindAction, Name: "page"}
lookup := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
cases := []struct {
name, owner string
run func(context.Context) error
allowed bool
}{
{"admin runs an action", alice, callTool(m, d, action), true},
{"viewer can't run an action", bob, callTool(m, d, action), false},
{"viewer looks up", bob, callTool(m, d, lookup), true},
{"admin saves a memory", alice, builtin(m, d, "memory_save"), true},
{"viewer can't save a memory", bob, builtin(m, d, "memory_save"), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ctx, err := m.actingAs(t.Context(), d, tc.owner)
if err != nil {
t.Fatal(err)
}
if err := tc.run(ctx); (err == nil) != tc.allowed {
t.Fatalf("allowed = %v (err %v), want %v", err == nil, err, tc.allowed)
}
})
}
}
// TestToolDiscoveryFollowsThePerson guards the discovery boundary: the model
// is offered only the tools its person may use, and an omitted tool's name and
// description appear nowhere in the prompt summaries.
func TestToolDiscoveryFollowsThePerson(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
turn := store.Turn{ID: uuid.New(), Owner: alice}
toolsFor := func(owner string) core.ToolSet {
t.Helper()
ctx, err := m.actingAs(t.Context(), d, owner)
if err != nil {
t.Fatal(err)
}
return m.tools(ctx, d, turn)
}
admin := toolsFor(alice)
for _, name := range []string{"pager_page", "pager_who", "dashboard_navigate", "memory_save"} {
if _, ok := admin[name]; !ok {
t.Fatalf("an admin isn't offered %q: %v", name, slices.Sorted(maps.Keys(admin)))
}
}
viewer := toolsFor(bob)
if _, ok := viewer["pager_page"]; ok {
t.Fatal("a viewer is offered the connector action")
}
if _, ok := viewer["pager_who"]; !ok {
t.Fatal("a viewer isn't offered the connector lookup")
}
if _, ok := viewer["memory_save"]; ok {
t.Fatal("a viewer is offered memory_save")
}
for _, name := range []string{"run_list", "dashboard_navigate"} {
if _, ok := viewer[name]; !ok {
t.Fatalf("a viewer isn't offered %q", name)
}
}
for _, summary := range toolSummaries(viewer) {
if summary.Name == "pager_page" || summary.Description == "Page the on-call person" {
t.Fatalf("an omitted tool is described in the prompt: %+v", summary)
}
}
// The next turn reads the person's role again, so a narrower role gets the
// narrower set without a restart.
people[bob] = person(bob, governance.RoleViewer)
if _, ok := toolsFor(bob)["pager_who"]; !ok {
t.Fatal("a viewer lost the lookup it may still use")
}
people[bob] = person(bob, governance.RoleOwner)
if _, ok := toolsFor(bob)["pager_page"]; !ok {
t.Fatal("a promoted person isn't offered the action on the next turn")
}
}
// unavailableAccess refuses every decision as an unloaded policy chain does.
type unavailableAccess struct{}
func (unavailableAccess) Authorize(context.Context, governance.Principal, governance.Action, governance.Resource, governance.AccessContext) governance.Decision {
return governance.Unavailable()
}
// TestToolDiscoveryFailsClosed guards the unavailable-policy case: no
// connector tool and no guarded builtin is offered, while navigation, which
// grants no access, remains.
func TestToolDiscoveryFailsClosed(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
d := Deps{People: people, Access: unavailableAccess{}, Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice})
for _, name := range []string{"pager_page", "pager_who", "run_list", "memory_list", "memory_save"} {
if _, ok := tools[name]; ok {
t.Fatalf("%q is offered while access policies are unavailable", name)
}
}
if _, ok := tools["dashboard_navigate"]; !ok {
t.Fatal("navigation is withheld")
}
}
// countingTool records whether anything was run.
type countingTool struct {
connectors.Tool
lookups int
}
func (c *countingTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) {
c.lookups++
return "someone", true, nil
}
// staleBox declares nothing, as a connector that stopped after the tool was
// offered.
type staleBox struct{ tool *countingTool }
func (b staleBox) Declarations() []connectors.Declaration { return nil }
func (b staleBox) Tool(string) (connectors.Tool, bool) { return b.tool, true }
// TestOmittedToolCannotRun guards invocation: a tool the registry no longer
// declares is refused before anything runs, and names nothing new.
func TestOmittedToolCannotRun(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
box := staleBox{tool: &countingTool{}}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
decl := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
if _, err := m.call(ctx, d, store.Turn{ID: uuid.New()}, decl, jsontext.Value(`{}`)); !errors.Is(err, errToolUnavailable) {
t.Fatalf("call error = %v, want %v", err, errToolUnavailable)
}
if box.tool.lookups != 0 {
t.Fatalf("a withheld tool ran %d times", box.tool.lookups)
}
}
// countingProvider records each model request the guard lets through.
type countingProvider struct{ calls int }
func (p *countingProvider) Name() string { return "counting" }
func (p *countingProvider) Chat(context.Context, chat.Request) (chat.Response, error) {
p.calls++
return chat.Response{}, nil
}
func (p *countingProvider) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
p.calls++
return nil, nil
}
// TestGuardStopsATurnWhenAccessIsWithdrawn guards the per-request recheck: the
// tool list is fixed for the turn, so a person who loses a tool between model
// requests stops the turn before the next request describes the stale set.
func TestGuardStopsATurnWhenAccessIsWithdrawn(t *testing.T) {
people := fakePeople{bob: person(bob, governance.RoleAdmin)}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
turn := store.Turn{ID: uuid.New(), Owner: bob}
ctx, err := m.actingAs(t.Context(), d, bob)
if err != nil {
t.Fatal(err)
}
offered := m.tools(ctx, d, turn)
provider := &countingProvider{}
guard := guardedProvider{Provider: provider, m: m, d: d, t: turn, offered: offered}
if _, err := guard.ChatStream(t.Context(), chat.Request{}); err != nil || provider.calls != 1 {
t.Fatalf("unchanged access: err %v, calls %d; want the request sent", err, provider.calls)
}
people[bob] = person(bob, governance.RoleViewer)
before := provider.calls
if _, err := guard.ChatStream(t.Context(), chat.Request{}); !errors.Is(err, errToolsChanged) {
t.Fatalf("withdrawn access: error = %v, want %v", err, errToolsChanged)
}
if provider.calls != before {
t.Fatal("a request was sent after a tool was withdrawn")
}
}
// TestToolNameCollisionsAreWithheld guards two declarations that collapse to
// one tool name: neither is offered, so a model call can't pick the wrong one.
func TestToolNameCollisionsAreWithheld(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
box := collisionBox{}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice})
if _, ok := tools["a_b_c"]; ok {
t.Fatal("two declarations with one tool name are offered")
}
if _, ok := tools["run_list"]; ok {
t.Fatal("a connector tool shadowing the dashboard read is offered")
}
}
type collisionBox struct{}
func (collisionBox) Declarations() []connectors.Declaration {
return []connectors.Declaration{
{Connector: "a_b", Kind: connectors.KindLookup, Name: "c"},
{Connector: "a", Kind: connectors.KindLookup, Name: "b_c"},
{Connector: "run", Kind: connectors.KindLookup, Name: "list"},
}
}
func (collisionBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
func callTool(m *Module, d Deps, decl connectors.Declaration) func(context.Context) error {
return func(ctx context.Context) error {
_, err := m.call(ctx, d, store.Turn{ID: uuid.New()}, decl, jsontext.Value(`{}`))
return err
}
}
func builtin(m *Module, d Deps, name string) func(context.Context) error {
return func(ctx context.Context) error {
for _, tool := range m.builtins(d, store.Turn{ID: uuid.New()}) {
if tool.tool.Name == name {
_, err := tool.tool.Execute(ctx, `{"text":"a fact"}`)
return err
}
}
return errors.New("no tool " + name)
}
}
// chatServer serves the chat API over a store holding one finished turn in a
// conversation alice owns. The X-As header names who calls.
func chatServer(t *testing.T, session, turn uuid.UUID) *httptest.Server {
t.Helper()
fs := &fakeStore{
sessions: map[uuid.UUID]store.Session{session: {ID: session, Owner: alice}},
turns: map[uuid.UUID]store.Turn{turn: {ID: turn, SessionID: session, Owner: alice, State: stateSucceeded}},
events: map[uuid.UUID][]store.Event{turn: {{Seq: 1, TurnID: turn, Kind: kindDone, Data: jsontext.Value(`{}`)}}},
}
m := &Module{
orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New(),
wake: make(chan struct{}, 1), running: map[uuid.UUID]context.CancelFunc{},
}
mux := http.NewServeMux()
m.Register(httpapi.New(mux, "test", headerAuth{}))
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func call(t *testing.T, srv *httptest.Server, as, method, path, body string) (int, string) {
t.Helper()
req, err := http.NewRequestWithContext(t.Context(), method, srv.URL+path, strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
req.Header.Set("X-As", as)
if body != "" {
req.Header.Set("Content-Type", "application/json")
}
res, err := srv.Client().Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = res.Body.Close() }()
out, _ := io.ReadAll(res.Body)
return res.StatusCode, string(out)
}
// headerAuth signs in whoever the X-As header names, as a viewer.
type headerAuth struct{}
func (headerAuth) Authenticate(ctx context.Context, r *http.Request) (context.Context, bool, error) {
as := r.Header.Get("X-As")
if as == "" {
return nil, false, httpapi.ErrUnauthenticated
}
return governance.WithPrincipal(ctx, person(as, governance.RoleViewer)), false, nil
}
type fakePeople map[string]governance.Principal
func (f fakePeople) PrincipalFor(_ context.Context, id governance.IdentityID) (governance.Principal, error) {
p, ok := f[string(id)]
if !ok {
return governance.Principal{}, errors.New("no such person")
}
return p, nil
}
type fakeToolbox struct{ Toolbox }
func (fakeToolbox) Declarations() []connectors.Declaration {
return []connectors.Declaration{
{Connector: "pager", Kind: connectors.KindAction, Name: "page", Description: "Page the on-call person", Schema: jsontext.Value(`{"type":"object"}`)},
{Connector: "pager", Kind: connectors.KindLookup, Name: "who", Description: "Who is on call", Schema: jsontext.Value(`{"type":"object"}`)},
}
}
func (fakeToolbox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
type fakeTool struct{ connectors.Tool }
func (fakeTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) {
return "someone", true, nil
}
type fakeJudge struct{ Judge }
func (fakeJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
return judgement.Decision{Key: req.Key, Level: judgement.ActQuietly}, nil
}
func (fakeJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
return "ran", nil
}
type fakeMemory struct{ Memory }
func (fakeMemory) Remember(context.Context, string) (memory.Note, error) { return memory.Note{}, nil }
// fakeStore keeps conversations in memory with the store's ownership rule:
// a read or change for an owner finds only that owner's conversations.
type fakeStore struct {
conversations
sessions map[uuid.UUID]store.Session
turns map[uuid.UUID]store.Turn
events map[uuid.UUID][]store.Event
}
func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) {
var out []store.Session
for _, s := range f.sessions {
if s.Owner == owner {
out = append(out, s)
}
}
return out, nil
}
func (f *fakeStore) Session(_ context.Context, _, owner string, id uuid.UUID) (store.Session, error) {
s, ok := f.sessions[id]
if !ok || s.Owner != owner {
return store.Session{}, store.ErrNotFound
}
return s, nil
}
func (f *fakeStore) Rename(ctx context.Context, org, owner string, id uuid.UUID, title string) error {
s, err := f.Session(ctx, org, owner, id)
if err == nil {
s.Title = title
f.sessions[id] = s
}
return err
}
func (f *fakeStore) DeleteSession(ctx context.Context, org, owner string, id uuid.UUID) error {
if _, err := f.Session(ctx, org, owner, id); err != nil {
return err
}
delete(f.sessions, id)
return nil
}
func (f *fakeStore) AddTurn(ctx context.Context, org, owner string, t store.Turn) (store.Turn, error) {
if _, err := f.Session(ctx, org, owner, t.SessionID); err != nil {
return store.Turn{}, err
}
t.Owner, t.State = owner, stateQueued
f.turns[t.ID] = t
return t, nil
}
func (f *fakeStore) Turns(_ context.Context, _, owner string, session uuid.UUID) ([]store.Turn, error) {
var out []store.Turn
for _, t := range f.turns {
if t.SessionID == session && t.Owner == owner {
out = append(out, t)
}
}
return out, nil
}
func (f *fakeStore) Turn(_ context.Context, _, owner string, id uuid.UUID) (store.Turn, error) {
t, ok := f.turns[id]
if !ok || t.Owner != owner {
return store.Turn{}, store.ErrNotFound
}
return t, nil
}
func (f *fakeStore) RequestCancel(ctx context.Context, org, owner string, id uuid.UUID) (bool, error) {
_, err := f.Turn(ctx, org, owner, id)
return false, err
}
func (f *fakeStore) Events(_ context.Context, turn uuid.UUID, after int64) ([]store.Event, error) {
var out []store.Event
for _, e := range f.events[turn] {
if e.Seq > after {
out = append(out, e)
}
}
return out, nil
}
func (f *fakeStore) EventsOf(context.Context, uuid.UUID, []string) ([]store.Event, error) {
return nil, nil
}