package chat import ( "context" "encoding/json/jsontext" "errors" "io" "log/slog" "maps" "net/http" "net/http/httptest" "slices" "strings" "testing" "github.com/google/uuid" "github.com/samcharles93/ai-sdk/chat" "github.com/samcharles93/ai-sdk/core" "github.com/samcharles93/archie-core/internal/platform/httpapi" "github.com/samcharles93/archie-core/internal/platform/hub" "github.com/samcharles93/archie-core/modules/chat/internal/store" "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/governance" "github.com/samcharles93/archie-core/modules/judgement" "github.com/samcharles93/archie-core/modules/memory" ) const ( alice = "alice" bob = "bob" ) // person is a principal holding role in the operating org. func person(id string, role governance.Role) governance.Principal { return governance.Principal{ IdentityID: governance.IdentityID(id), Org: governance.DefaultOrgID, Memberships: []governance.Membership{{IdentityID: governance.IdentityID(id), OrgID: governance.DefaultOrgID, Role: role}}, } } // TestConversationsBelongToTheirOwner guards the chat API's ownership // boundary: a person reads and continues their own conversations, and every // chat route answers anyone else as if the conversation did not exist. func TestConversationsBelongToTheirOwner(t *testing.T) { session, turn := uuid.New(), uuid.New() cases := []struct { name, method, path, body string ok int }{ {"rename", http.MethodPatch, "/api/chat/sessions/" + session.String(), `{"title":"mine"}`, http.StatusOK}, {"list turns", http.MethodGet, "/api/chat/sessions/" + session.String() + "/turns", "", http.StatusOK}, {"send", http.MethodPost, "/api/chat/sessions/" + session.String() + "/turns", `{"text":"hello"}`, http.StatusAccepted}, {"get turn", http.MethodGet, "/api/chat/turns/" + turn.String(), "", http.StatusOK}, {"cancel", http.MethodPost, "/api/chat/turns/" + turn.String() + "/cancel", "", http.StatusOK}, {"stream", http.MethodGet, "/api/chat/turns/" + turn.String() + "/events", "", http.StatusOK}, {"delete", http.MethodDelete, "/api/chat/sessions/" + session.String(), "", http.StatusNoContent}, } for _, tc := range cases { for _, as := range []string{alice, bob} { t.Run(tc.name+" as "+as, func(t *testing.T) { srv := chatServer(t, session, turn) want := tc.ok if as != alice { want = http.StatusNotFound } if got, _ := call(t, srv, as, tc.method, tc.path, tc.body); got != want { t.Fatalf("%s %s as %s = %d, want %d", tc.method, tc.path, as, got, want) } }) } } t.Run("list sessions", func(t *testing.T) { srv := chatServer(t, session, turn) for as, sees := range map[string]bool{alice: true, bob: false} { code, body := call(t, srv, as, http.MethodGet, "/api/chat/sessions", "") if code != http.StatusOK || strings.Contains(body, session.String()) != sees { t.Fatalf("list as %s = %d %s, want it to show the conversation: %v", as, code, body, sees) } } }) } // TestTurnActsWithItsOwnersAccess guards the tool boundary: the same tool // call runs in an administrator's conversation and is refused in a viewer's, // so the agent never reaches further than the person chatting. func TestTurnActsWithItsOwnersAccess(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)} d := Deps{ People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{}, } m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()} action := connectors.Declaration{Connector: "pager", Privileged: noPrivilegedEffects, Kind: connectors.KindAction, Name: "page"} lookup := connectors.Declaration{Connector: "pager", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "who"} cases := []struct { name, owner string run func(context.Context) error allowed bool }{ {"admin runs an action", alice, callTool(m, d, action), true}, {"viewer can't run an action", bob, callTool(m, d, action), false}, {"viewer looks up", bob, callTool(m, d, lookup), true}, {"admin saves a memory", alice, builtin(m, d, "memory_save"), true}, {"viewer can't save a memory", bob, builtin(m, d, "memory_save"), false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ctx, err := m.actingAs(t.Context(), d, tc.owner) if err != nil { t.Fatal(err) } if err := tc.run(ctx); (err == nil) != tc.allowed { t.Fatalf("allowed = %v (err %v), want %v", err == nil, err, tc.allowed) } }) } } // TestToolDiscoveryFollowsThePerson guards the discovery boundary: the model // is offered only the tools its person may use, and an omitted tool's name and // description appear nowhere in the prompt summaries. func TestToolDiscoveryFollowsThePerson(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}} m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} turn := store.Turn{ID: uuid.New(), Owner: alice} toolsFor := func(owner string) core.ToolSet { t.Helper() ctx, err := m.actingAs(t.Context(), d, owner) if err != nil { t.Fatal(err) } return m.tools(ctx, d, turn) } admin := toolsFor(alice) for _, name := range []string{"pager_page", "pager_who", "dashboard_navigate", "memory_save"} { if _, ok := admin[name]; !ok { t.Fatalf("an admin isn't offered %q: %v", name, slices.Sorted(maps.Keys(admin))) } } viewer := toolsFor(bob) if _, ok := viewer["pager_page"]; ok { t.Fatal("a viewer is offered the connector action") } if _, ok := viewer["pager_who"]; !ok { t.Fatal("a viewer isn't offered the connector lookup") } if _, ok := viewer["memory_save"]; ok { t.Fatal("a viewer is offered memory_save") } for _, name := range []string{"run_list", "dashboard_navigate"} { if _, ok := viewer[name]; !ok { t.Fatalf("a viewer isn't offered %q", name) } } for _, summary := range toolSummaries(viewer) { if summary.Name == "pager_page" || summary.Description == "Page the on-call person" { t.Fatalf("an omitted tool is described in the prompt: %+v", summary) } } // The next turn reads the person's role again, so a narrower role gets the // narrower set without a restart. people[bob] = person(bob, governance.RoleViewer) if _, ok := toolsFor(bob)["pager_who"]; !ok { t.Fatal("a viewer lost the lookup it may still use") } people[bob] = person(bob, governance.RoleOwner) if _, ok := toolsFor(bob)["pager_page"]; !ok { t.Fatal("a promoted person isn't offered the action on the next turn") } } // unavailableAccess refuses every decision as an unloaded policy chain does. type unavailableAccess struct{} func (unavailableAccess) Authorize(context.Context, governance.Principal, governance.Action, governance.Resource, governance.AccessContext) governance.Decision { return governance.Unavailable() } // TestToolDiscoveryFailsClosed guards the unavailable-policy case: no // connector tool and no guarded builtin is offered, while navigation, which // grants no access, remains. func TestToolDiscoveryFailsClosed(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin)} d := Deps{People: people, Access: unavailableAccess{}, Toolbox: fakeToolbox{}} m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} ctx, err := m.actingAs(t.Context(), d, alice) if err != nil { t.Fatal(err) } tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice}) for _, name := range []string{"pager_page", "pager_who", "run_list", "memory_list", "memory_save"} { if _, ok := tools[name]; ok { t.Fatalf("%q is offered while access policies are unavailable", name) } } if _, ok := tools["dashboard_navigate"]; !ok { t.Fatal("navigation is withheld") } } // countingTool records whether anything was run. type countingTool struct { connectors.Tool lookups int } func (c *countingTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) { c.lookups++ return "someone", true, nil } // staleBox declares nothing, as a connector that stopped after the tool was // offered. type staleBox struct{ tool *countingTool } func (b staleBox) Declarations() []connectors.Declaration { return nil } func (b staleBox) Tool(string) (connectors.Tool, bool) { return b.tool, true } // TestOmittedToolCannotRun guards invocation: a tool the registry no longer // declares is refused before anything runs, and names nothing new. func TestOmittedToolCannotRun(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin)} box := staleBox{tool: &countingTool{}} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box} m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()} ctx, err := m.actingAs(t.Context(), d, alice) if err != nil { t.Fatal(err) } decl := connectors.Declaration{Connector: "pager", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "who"} if _, err := m.call(ctx, d, store.Turn{ID: uuid.New()}, decl, jsontext.Value(`{}`)); !errors.Is(err, errToolUnavailable) { t.Fatalf("call error = %v, want %v", err, errToolUnavailable) } if box.tool.lookups != 0 { t.Fatalf("a withheld tool ran %d times", box.tool.lookups) } } // countingProvider records each model request the guard lets through. type countingProvider struct{ calls int } func (p *countingProvider) Name() string { return "counting" } func (p *countingProvider) Chat(context.Context, chat.Request) (chat.Response, error) { p.calls++ return chat.Response{}, nil } func (p *countingProvider) ChatStream(context.Context, chat.Request) (chat.Stream, error) { p.calls++ return nil, nil } // TestGuardStopsATurnWhenAccessIsWithdrawn guards the per-request recheck: the // tool list is fixed for the turn, so a person who loses a tool between model // requests stops the turn before the next request describes the stale set. func TestGuardStopsATurnWhenAccessIsWithdrawn(t *testing.T) { people := fakePeople{bob: person(bob, governance.RoleAdmin)} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}} m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} turn := store.Turn{ID: uuid.New(), Owner: bob} ctx, err := m.actingAs(t.Context(), d, bob) if err != nil { t.Fatal(err) } offered := m.tools(ctx, d, turn) provider := &countingProvider{} guard := guardedProvider{Provider: provider, m: m, d: d, t: turn, offered: offered} if _, err := guard.ChatStream(t.Context(), chat.Request{}); err != nil || provider.calls != 1 { t.Fatalf("unchanged access: err %v, calls %d; want the request sent", err, provider.calls) } people[bob] = person(bob, governance.RoleViewer) before := provider.calls if _, err := guard.ChatStream(t.Context(), chat.Request{}); !errors.Is(err, errToolsChanged) { t.Fatalf("withdrawn access: error = %v, want %v", err, errToolsChanged) } if provider.calls != before { t.Fatal("a request was sent after a tool was withdrawn") } } // TestToolNameCollisionsAreWithheld guards two declarations that collapse to // one tool name: neither is offered, so a model call can't pick the wrong one. func TestToolNameCollisionsAreWithheld(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleAdmin)} box := collisionBox{} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box} m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()} ctx, err := m.actingAs(t.Context(), d, alice) if err != nil { t.Fatal(err) } tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice}) if _, ok := tools["a_b_c"]; ok { t.Fatal("two declarations with one tool name are offered") } if _, ok := tools["run_list"]; ok { t.Fatal("a connector tool shadowing the dashboard read is offered") } } type collisionBox struct{} func (collisionBox) Declarations() []connectors.Declaration { return []connectors.Declaration{ {Connector: "a_b", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "c"}, {Connector: "a", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "b_c"}, {Connector: "run", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "list"}, } } func (collisionBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true } // freshTurn is a turn in a conversation of its own. The taint flag is kept per // conversation, so a turn sharing the zero session with another case would // inherit that case's taint. func freshTurn() store.Turn { return store.Turn{ID: uuid.New(), SessionID: uuid.New()} } func callTool(m *Module, d Deps, decl connectors.Declaration) func(context.Context) error { return func(ctx context.Context) error { _, err := m.call(ctx, d, freshTurn(), decl, jsontext.Value(`{}`)) return err } } func builtin(m *Module, d Deps, name string) func(context.Context) error { return func(ctx context.Context) error { for _, tool := range m.builtins(d, freshTurn()) { if tool.tool.Name == name { _, err := tool.tool.Execute(ctx, `{"text":"a fact"}`) return err } } return errors.New("no tool " + name) } } // chatServer serves the chat API over a store holding one finished turn in a // conversation alice owns. The X-As header names who calls. func chatServer(t *testing.T, session, turn uuid.UUID) *httptest.Server { t.Helper() fs := &fakeStore{ sessions: map[uuid.UUID]store.Session{session: {ID: session, Owner: alice}}, turns: map[uuid.UUID]store.Turn{turn: {ID: turn, SessionID: session, Owner: alice, State: stateSucceeded}}, events: map[uuid.UUID][]store.Event{turn: {{Seq: 1, TurnID: turn, Kind: kindDone, Data: jsontext.Value(`{}`)}}}, } m := &Module{ orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New(), wake: make(chan struct{}, 1), running: map[uuid.UUID]context.CancelFunc{}, } mux := http.NewServeMux() m.Register(httpapi.New(mux, "test", headerAuth{}, allowAll{})) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) return srv } func call(t *testing.T, srv *httptest.Server, as, method, path, body string) (int, string) { t.Helper() req, err := http.NewRequestWithContext(t.Context(), method, srv.URL+path, strings.NewReader(body)) if err != nil { t.Fatal(err) } req.Header.Set("X-As", as) if body != "" { req.Header.Set("Content-Type", "application/json") } res, err := srv.Client().Do(req) if err != nil { t.Fatal(err) } defer func() { _ = res.Body.Close() }() out, _ := io.ReadAll(res.Body) return res.StatusCode, string(out) } // headerAuth signs in whoever the X-As header names, as a viewer. type headerAuth struct{} func (headerAuth) Authenticate(ctx context.Context, r *http.Request) (context.Context, bool, error) { as := r.Header.Get("X-As") if as == "" { return nil, false, httpapi.ErrUnauthenticated } return governance.WithPrincipal(ctx, person(as, governance.RoleViewer)), false, nil } type fakePeople map[string]governance.Principal func (f fakePeople) PrincipalFor(_ context.Context, id governance.IdentityID) (governance.Principal, error) { p, ok := f[string(id)] if !ok { return governance.Principal{}, errors.New("no such person") } return p, nil } type fakeToolbox struct{ Toolbox } func (fakeToolbox) Declarations() []connectors.Declaration { return []connectors.Declaration{ {Connector: "pager", Privileged: noPrivilegedEffects, Kind: connectors.KindAction, Name: "page", Description: "Page the on-call person", Schema: jsontext.Value(`{"type":"object"}`)}, {Connector: "pager", Privileged: noPrivilegedEffects, Kind: connectors.KindLookup, Name: "who", Description: "Who is on call", Schema: jsontext.Value(`{"type":"object"}`)}, } } func (fakeToolbox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true } type fakeTool struct{ connectors.Tool } func (fakeTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) { return "someone", true, nil } type fakeJudge struct{ Judge } func (fakeJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) { return judgement.Decision{Key: req.Key, Level: judgement.ActQuietly}, nil } func (fakeJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) { return "ran", nil } type fakeMemory struct{ Memory } func (fakeMemory) Notes(context.Context, string, memory.Class) ([]memory.Note, error) { return nil, nil } func (fakeMemory) Remember(context.Context, string, string) (memory.Note, error) { return memory.Note{}, nil } func (fakeMemory) Forget(context.Context, string, uuid.UUID) (bool, error) { return true, nil } func (fakeMemory) Propose(context.Context, string, string) (memory.Note, error) { return memory.Note{}, nil } // fakeStore keeps conversations in memory with the store's ownership rule: // a read or change for an owner finds only that owner's conversations. type fakeStore struct { conversations sessions map[uuid.UUID]store.Session turns map[uuid.UUID]store.Turn events map[uuid.UUID][]store.Event tainted map[uuid.UUID]bool actions map[string]store.Action } func (f *fakeStore) Taint(_ context.Context, _, _ string, id uuid.UUID) error { if f.tainted == nil { f.tainted = map[uuid.UUID]bool{} } f.tainted[id] = true return nil } func (f *fakeStore) SessionTainted(_ context.Context, _, _ string, id uuid.UUID) (bool, error) { return f.tainted[id], nil } func (f *fakeStore) PutAction(_ context.Context, _ string, a store.Action) (store.Action, error) { if f.actions == nil { f.actions = map[string]store.Action{} } f.actions[a.TurnID.String()+":"+a.Key] = a return a, nil } func (f *fakeStore) AddEvent(_ context.Context, turn uuid.UUID, kind string, data jsontext.Value) (int64, error) { if f.events == nil { f.events = map[uuid.UUID][]store.Event{} } seq := int64(len(f.events[turn]) + 1) f.events[turn] = append(f.events[turn], store.Event{Seq: seq, TurnID: turn, Kind: kind, Data: data}) return seq, nil } func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) { var out []store.Session for _, s := range f.sessions { if s.Owner == owner { out = append(out, s) } } return out, nil } func (f *fakeStore) Session(_ context.Context, _, owner string, id uuid.UUID) (store.Session, error) { s, ok := f.sessions[id] if !ok || s.Owner != owner { return store.Session{}, store.ErrNotFound } return s, nil } func (f *fakeStore) Rename(ctx context.Context, org, owner string, id uuid.UUID, title string) error { s, err := f.Session(ctx, org, owner, id) if err == nil { s.Title = title f.sessions[id] = s } return err } func (f *fakeStore) DeleteSession(ctx context.Context, org, owner string, id uuid.UUID) error { if _, err := f.Session(ctx, org, owner, id); err != nil { return err } delete(f.sessions, id) return nil } func (f *fakeStore) AddTurn(ctx context.Context, org, owner string, t store.Turn) (store.Turn, error) { if _, err := f.Session(ctx, org, owner, t.SessionID); err != nil { return store.Turn{}, err } t.Owner, t.State = owner, stateQueued f.turns[t.ID] = t return t, nil } func (f *fakeStore) Turns(_ context.Context, _, owner string, session uuid.UUID) ([]store.Turn, error) { var out []store.Turn for _, t := range f.turns { if t.SessionID == session && t.Owner == owner { out = append(out, t) } } return out, nil } func (f *fakeStore) Turn(_ context.Context, _, owner string, id uuid.UUID) (store.Turn, error) { t, ok := f.turns[id] if !ok || t.Owner != owner { return store.Turn{}, store.ErrNotFound } return t, nil } func (f *fakeStore) RequestCancel(ctx context.Context, org, owner string, id uuid.UUID) (bool, error) { _, err := f.Turn(ctx, org, owner, id) return false, err } func (f *fakeStore) Events(_ context.Context, turn uuid.UUID, after int64) ([]store.Event, error) { var out []store.Event for _, e := range f.events[turn] { if e.Seq > after { out = append(out, e) } } return out, nil } func (f *fakeStore) EventsOf(context.Context, uuid.UUID, []string) ([]store.Event, error) { return nil, nil } // noPrivilegedEffects is a declaration that states it has no privileged effect, // the only kind chat offers. var noPrivilegedEffects = connectors.PrivilegedEffects{Declared: true} // allowAll leaves every decision to the handlers under test. type allowAll struct{} func (allowAll) Allow(context.Context, string, string) error { return nil }