Files
archie/modules/chat/access_test.go
T
sam 073959b4f9 feat(memory): scope chat notes and preserve deletions on restore (#2069)
Keep personal notes private by default and require review before publishing shared notes. Restore backups and replay forgotten notes in one transaction, and retain the installed release when an unattended rollback could bring back forgotten notes.
2026-11-21 13:20:31 +11:00

568 lines
20 KiB
Go

package chat
import (
"context"
"encoding/json/jsontext"
"errors"
"io"
"log/slog"
"maps"
"net/http"
"net/http/httptest"
"slices"
"strings"
"testing"
"github.com/google/uuid"
"github.com/samcharles93/ai-sdk/chat"
"github.com/samcharles93/ai-sdk/core"
"github.com/samcharles93/archie-core/internal/platform/httpapi"
"github.com/samcharles93/archie-core/internal/platform/hub"
"github.com/samcharles93/archie-core/modules/chat/internal/store"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/governance"
"github.com/samcharles93/archie-core/modules/judgement"
"github.com/samcharles93/archie-core/modules/memory"
)
const (
alice = "alice"
bob = "bob"
)
// person is a principal holding role in the operating org.
func person(id string, role governance.Role) governance.Principal {
return governance.Principal{
IdentityID: governance.IdentityID(id), Org: governance.DefaultOrgID,
Memberships: []governance.Membership{{IdentityID: governance.IdentityID(id), OrgID: governance.DefaultOrgID, Role: role}},
}
}
// TestConversationsBelongToTheirOwner guards the chat API's ownership
// boundary: a person reads and continues their own conversations, and every
// chat route answers anyone else as if the conversation did not exist.
func TestConversationsBelongToTheirOwner(t *testing.T) {
session, turn := uuid.New(), uuid.New()
cases := []struct {
name, method, path, body string
ok int
}{
{"rename", http.MethodPatch, "/api/chat/sessions/" + session.String(), `{"title":"mine"}`, http.StatusOK},
{"list turns", http.MethodGet, "/api/chat/sessions/" + session.String() + "/turns", "", http.StatusOK},
{"send", http.MethodPost, "/api/chat/sessions/" + session.String() + "/turns", `{"text":"hello"}`, http.StatusAccepted},
{"get turn", http.MethodGet, "/api/chat/turns/" + turn.String(), "", http.StatusOK},
{"cancel", http.MethodPost, "/api/chat/turns/" + turn.String() + "/cancel", "", http.StatusOK},
{"stream", http.MethodGet, "/api/chat/turns/" + turn.String() + "/events", "", http.StatusOK},
{"delete", http.MethodDelete, "/api/chat/sessions/" + session.String(), "", http.StatusNoContent},
}
for _, tc := range cases {
for _, as := range []string{alice, bob} {
t.Run(tc.name+" as "+as, func(t *testing.T) {
srv := chatServer(t, session, turn)
want := tc.ok
if as != alice {
want = http.StatusNotFound
}
if got, _ := call(t, srv, as, tc.method, tc.path, tc.body); got != want {
t.Fatalf("%s %s as %s = %d, want %d", tc.method, tc.path, as, got, want)
}
})
}
}
t.Run("list sessions", func(t *testing.T) {
srv := chatServer(t, session, turn)
for as, sees := range map[string]bool{alice: true, bob: false} {
code, body := call(t, srv, as, http.MethodGet, "/api/chat/sessions", "")
if code != http.StatusOK || strings.Contains(body, session.String()) != sees {
t.Fatalf("list as %s = %d %s, want it to show the conversation: %v", as, code, body, sees)
}
}
})
}
// TestTurnActsWithItsOwnersAccess guards the tool boundary: the same tool
// call runs in an administrator's conversation and is refused in a viewer's,
// so the agent never reaches further than the person chatting.
func TestTurnActsWithItsOwnersAccess(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)}
d := Deps{
People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)),
Toolbox: fakeToolbox{}, Judge: fakeJudge{}, Memory: fakeMemory{},
}
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
action := connectors.Declaration{Connector: "pager", Kind: connectors.KindAction, Name: "page"}
lookup := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
cases := []struct {
name, owner string
run func(context.Context) error
allowed bool
}{
{"admin runs an action", alice, callTool(m, d, action), true},
{"viewer can't run an action", bob, callTool(m, d, action), false},
{"viewer looks up", bob, callTool(m, d, lookup), true},
{"admin saves a memory", alice, builtin(m, d, "memory_save"), true},
{"viewer can't save a memory", bob, builtin(m, d, "memory_save"), false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ctx, err := m.actingAs(t.Context(), d, tc.owner)
if err != nil {
t.Fatal(err)
}
if err := tc.run(ctx); (err == nil) != tc.allowed {
t.Fatalf("allowed = %v (err %v), want %v", err == nil, err, tc.allowed)
}
})
}
}
// TestToolDiscoveryFollowsThePerson guards the discovery boundary: the model
// is offered only the tools its person may use, and an omitted tool's name and
// description appear nowhere in the prompt summaries.
func TestToolDiscoveryFollowsThePerson(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin), bob: person(bob, governance.RoleViewer)}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
turn := store.Turn{ID: uuid.New(), Owner: alice}
toolsFor := func(owner string) core.ToolSet {
t.Helper()
ctx, err := m.actingAs(t.Context(), d, owner)
if err != nil {
t.Fatal(err)
}
return m.tools(ctx, d, turn)
}
admin := toolsFor(alice)
for _, name := range []string{"pager_page", "pager_who", "dashboard_navigate", "memory_save"} {
if _, ok := admin[name]; !ok {
t.Fatalf("an admin isn't offered %q: %v", name, slices.Sorted(maps.Keys(admin)))
}
}
viewer := toolsFor(bob)
if _, ok := viewer["pager_page"]; ok {
t.Fatal("a viewer is offered the connector action")
}
if _, ok := viewer["pager_who"]; !ok {
t.Fatal("a viewer isn't offered the connector lookup")
}
if _, ok := viewer["memory_save"]; ok {
t.Fatal("a viewer is offered memory_save")
}
for _, name := range []string{"run_list", "dashboard_navigate"} {
if _, ok := viewer[name]; !ok {
t.Fatalf("a viewer isn't offered %q", name)
}
}
for _, summary := range toolSummaries(viewer) {
if summary.Name == "pager_page" || summary.Description == "Page the on-call person" {
t.Fatalf("an omitted tool is described in the prompt: %+v", summary)
}
}
// The next turn reads the person's role again, so a narrower role gets the
// narrower set without a restart.
people[bob] = person(bob, governance.RoleViewer)
if _, ok := toolsFor(bob)["pager_who"]; !ok {
t.Fatal("a viewer lost the lookup it may still use")
}
people[bob] = person(bob, governance.RoleOwner)
if _, ok := toolsFor(bob)["pager_page"]; !ok {
t.Fatal("a promoted person isn't offered the action on the next turn")
}
}
// unavailableAccess refuses every decision as an unloaded policy chain does.
type unavailableAccess struct{}
func (unavailableAccess) Authorize(context.Context, governance.Principal, governance.Action, governance.Resource, governance.AccessContext) governance.Decision {
return governance.Unavailable()
}
// TestToolDiscoveryFailsClosed guards the unavailable-policy case: no
// connector tool and no guarded builtin is offered, while navigation, which
// grants no access, remains.
func TestToolDiscoveryFailsClosed(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
d := Deps{People: people, Access: unavailableAccess{}, Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice})
for _, name := range []string{"pager_page", "pager_who", "run_list", "memory_list", "memory_save"} {
if _, ok := tools[name]; ok {
t.Fatalf("%q is offered while access policies are unavailable", name)
}
}
if _, ok := tools["dashboard_navigate"]; !ok {
t.Fatal("navigation is withheld")
}
}
// countingTool records whether anything was run.
type countingTool struct {
connectors.Tool
lookups int
}
func (c *countingTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) {
c.lookups++
return "someone", true, nil
}
// staleBox declares nothing, as a connector that stopped after the tool was
// offered.
type staleBox struct{ tool *countingTool }
func (b staleBox) Declarations() []connectors.Declaration { return nil }
func (b staleBox) Tool(string) (connectors.Tool, bool) { return b.tool, true }
// TestOmittedToolCannotRun guards invocation: a tool the registry no longer
// declares is refused before anything runs, and names nothing new.
func TestOmittedToolCannotRun(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
box := staleBox{tool: &countingTool{}}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
m := &Module{orgID: "default", store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
decl := connectors.Declaration{Connector: "pager", Kind: connectors.KindLookup, Name: "who"}
if _, err := m.call(ctx, d, store.Turn{ID: uuid.New()}, decl, jsontext.Value(`{}`)); !errors.Is(err, errToolUnavailable) {
t.Fatalf("call error = %v, want %v", err, errToolUnavailable)
}
if box.tool.lookups != 0 {
t.Fatalf("a withheld tool ran %d times", box.tool.lookups)
}
}
// countingProvider records each model request the guard lets through.
type countingProvider struct{ calls int }
func (p *countingProvider) Name() string { return "counting" }
func (p *countingProvider) Chat(context.Context, chat.Request) (chat.Response, error) {
p.calls++
return chat.Response{}, nil
}
func (p *countingProvider) ChatStream(context.Context, chat.Request) (chat.Stream, error) {
p.calls++
return nil, nil
}
// TestGuardStopsATurnWhenAccessIsWithdrawn guards the per-request recheck: the
// tool list is fixed for the turn, so a person who loses a tool between model
// requests stops the turn before the next request describes the stale set.
func TestGuardStopsATurnWhenAccessIsWithdrawn(t *testing.T) {
people := fakePeople{bob: person(bob, governance.RoleAdmin)}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: fakeToolbox{}}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
turn := store.Turn{ID: uuid.New(), Owner: bob}
ctx, err := m.actingAs(t.Context(), d, bob)
if err != nil {
t.Fatal(err)
}
offered := m.tools(ctx, d, turn)
provider := &countingProvider{}
guard := guardedProvider{Provider: provider, m: m, d: d, t: turn, offered: offered}
if _, err := guard.ChatStream(t.Context(), chat.Request{}); err != nil || provider.calls != 1 {
t.Fatalf("unchanged access: err %v, calls %d; want the request sent", err, provider.calls)
}
people[bob] = person(bob, governance.RoleViewer)
before := provider.calls
if _, err := guard.ChatStream(t.Context(), chat.Request{}); !errors.Is(err, errToolsChanged) {
t.Fatalf("withdrawn access: error = %v, want %v", err, errToolsChanged)
}
if provider.calls != before {
t.Fatal("a request was sent after a tool was withdrawn")
}
}
// TestToolNameCollisionsAreWithheld guards two declarations that collapse to
// one tool name: neither is offered, so a model call can't pick the wrong one.
func TestToolNameCollisionsAreWithheld(t *testing.T) {
people := fakePeople{alice: person(alice, governance.RoleAdmin)}
box := collisionBox{}
d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box}
m := &Module{orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New()}
ctx, err := m.actingAs(t.Context(), d, alice)
if err != nil {
t.Fatal(err)
}
tools := m.tools(ctx, d, store.Turn{ID: uuid.New(), Owner: alice})
if _, ok := tools["a_b_c"]; ok {
t.Fatal("two declarations with one tool name are offered")
}
if _, ok := tools["run_list"]; ok {
t.Fatal("a connector tool shadowing the dashboard read is offered")
}
}
type collisionBox struct{}
func (collisionBox) Declarations() []connectors.Declaration {
return []connectors.Declaration{
{Connector: "a_b", Kind: connectors.KindLookup, Name: "c"},
{Connector: "a", Kind: connectors.KindLookup, Name: "b_c"},
{Connector: "run", Kind: connectors.KindLookup, Name: "list"},
}
}
func (collisionBox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
// freshTurn is a turn in a conversation of its own. The taint flag is kept per
// conversation, so a turn sharing the zero session with another case would
// inherit that case's taint.
func freshTurn() store.Turn {
return store.Turn{ID: uuid.New(), SessionID: uuid.New()}
}
func callTool(m *Module, d Deps, decl connectors.Declaration) func(context.Context) error {
return func(ctx context.Context) error {
_, err := m.call(ctx, d, freshTurn(), decl, jsontext.Value(`{}`))
return err
}
}
func builtin(m *Module, d Deps, name string) func(context.Context) error {
return func(ctx context.Context) error {
for _, tool := range m.builtins(d, freshTurn()) {
if tool.tool.Name == name {
_, err := tool.tool.Execute(ctx, `{"text":"a fact"}`)
return err
}
}
return errors.New("no tool " + name)
}
}
// chatServer serves the chat API over a store holding one finished turn in a
// conversation alice owns. The X-As header names who calls.
func chatServer(t *testing.T, session, turn uuid.UUID) *httptest.Server {
t.Helper()
fs := &fakeStore{
sessions: map[uuid.UUID]store.Session{session: {ID: session, Owner: alice}},
turns: map[uuid.UUID]store.Turn{turn: {ID: turn, SessionID: session, Owner: alice, State: stateSucceeded}},
events: map[uuid.UUID][]store.Event{turn: {{Seq: 1, TurnID: turn, Kind: kindDone, Data: jsontext.Value(`{}`)}}},
}
m := &Module{
orgID: "default", store: fs, log: slog.New(slog.DiscardHandler), hub: hub.New(),
wake: make(chan struct{}, 1), running: map[uuid.UUID]context.CancelFunc{},
}
mux := http.NewServeMux()
m.Register(httpapi.New(mux, "test", headerAuth{}))
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
return srv
}
func call(t *testing.T, srv *httptest.Server, as, method, path, body string) (int, string) {
t.Helper()
req, err := http.NewRequestWithContext(t.Context(), method, srv.URL+path, strings.NewReader(body))
if err != nil {
t.Fatal(err)
}
req.Header.Set("X-As", as)
if body != "" {
req.Header.Set("Content-Type", "application/json")
}
res, err := srv.Client().Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = res.Body.Close() }()
out, _ := io.ReadAll(res.Body)
return res.StatusCode, string(out)
}
// headerAuth signs in whoever the X-As header names, as a viewer.
type headerAuth struct{}
func (headerAuth) Authenticate(ctx context.Context, r *http.Request) (context.Context, bool, error) {
as := r.Header.Get("X-As")
if as == "" {
return nil, false, httpapi.ErrUnauthenticated
}
return governance.WithPrincipal(ctx, person(as, governance.RoleViewer)), false, nil
}
type fakePeople map[string]governance.Principal
func (f fakePeople) PrincipalFor(_ context.Context, id governance.IdentityID) (governance.Principal, error) {
p, ok := f[string(id)]
if !ok {
return governance.Principal{}, errors.New("no such person")
}
return p, nil
}
type fakeToolbox struct{ Toolbox }
func (fakeToolbox) Declarations() []connectors.Declaration {
return []connectors.Declaration{
{Connector: "pager", Kind: connectors.KindAction, Name: "page", Description: "Page the on-call person", Schema: jsontext.Value(`{"type":"object"}`)},
{Connector: "pager", Kind: connectors.KindLookup, Name: "who", Description: "Who is on call", Schema: jsontext.Value(`{"type":"object"}`)},
}
}
func (fakeToolbox) Tool(string) (connectors.Tool, bool) { return fakeTool{}, true }
type fakeTool struct{ connectors.Tool }
func (fakeTool) Lookup(context.Context, string, jsontext.Value) (string, bool, error) {
return "someone", true, nil
}
type fakeJudge struct{ Judge }
func (fakeJudge) Assess(_ context.Context, req judgement.Request) (judgement.Decision, error) {
return judgement.Decision{Key: req.Key, Level: judgement.ActQuietly}, nil
}
func (fakeJudge) Execute(context.Context, judgement.Decision, jsontext.Value, string) (string, error) {
return "ran", nil
}
type fakeMemory struct{ Memory }
func (fakeMemory) Notes(context.Context, string, memory.Class) ([]memory.Note, error) {
return nil, nil
}
func (fakeMemory) Remember(context.Context, string, string) (memory.Note, error) {
return memory.Note{}, nil
}
func (fakeMemory) Forget(context.Context, string, uuid.UUID) (bool, error) { return true, nil }
func (fakeMemory) Propose(context.Context, string, string) (memory.Note, error) {
return memory.Note{}, nil
}
// fakeStore keeps conversations in memory with the store's ownership rule:
// a read or change for an owner finds only that owner's conversations.
type fakeStore struct {
conversations
sessions map[uuid.UUID]store.Session
turns map[uuid.UUID]store.Turn
events map[uuid.UUID][]store.Event
tainted map[uuid.UUID]bool
actions map[string]store.Action
}
func (f *fakeStore) Taint(_ context.Context, _, _ string, id uuid.UUID) error {
if f.tainted == nil {
f.tainted = map[uuid.UUID]bool{}
}
f.tainted[id] = true
return nil
}
func (f *fakeStore) SessionTainted(_ context.Context, _, _ string, id uuid.UUID) (bool, error) {
return f.tainted[id], nil
}
func (f *fakeStore) PutAction(_ context.Context, _ string, a store.Action) (store.Action, error) {
if f.actions == nil {
f.actions = map[string]store.Action{}
}
f.actions[a.TurnID.String()+":"+a.Key] = a
return a, nil
}
func (f *fakeStore) AddEvent(_ context.Context, turn uuid.UUID, kind string, data jsontext.Value) (int64, error) {
if f.events == nil {
f.events = map[uuid.UUID][]store.Event{}
}
seq := int64(len(f.events[turn]) + 1)
f.events[turn] = append(f.events[turn], store.Event{Seq: seq, TurnID: turn, Kind: kind, Data: data})
return seq, nil
}
func (f *fakeStore) Sessions(_ context.Context, _, owner string, _ int) ([]store.Session, error) {
var out []store.Session
for _, s := range f.sessions {
if s.Owner == owner {
out = append(out, s)
}
}
return out, nil
}
func (f *fakeStore) Session(_ context.Context, _, owner string, id uuid.UUID) (store.Session, error) {
s, ok := f.sessions[id]
if !ok || s.Owner != owner {
return store.Session{}, store.ErrNotFound
}
return s, nil
}
func (f *fakeStore) Rename(ctx context.Context, org, owner string, id uuid.UUID, title string) error {
s, err := f.Session(ctx, org, owner, id)
if err == nil {
s.Title = title
f.sessions[id] = s
}
return err
}
func (f *fakeStore) DeleteSession(ctx context.Context, org, owner string, id uuid.UUID) error {
if _, err := f.Session(ctx, org, owner, id); err != nil {
return err
}
delete(f.sessions, id)
return nil
}
func (f *fakeStore) AddTurn(ctx context.Context, org, owner string, t store.Turn) (store.Turn, error) {
if _, err := f.Session(ctx, org, owner, t.SessionID); err != nil {
return store.Turn{}, err
}
t.Owner, t.State = owner, stateQueued
f.turns[t.ID] = t
return t, nil
}
func (f *fakeStore) Turns(_ context.Context, _, owner string, session uuid.UUID) ([]store.Turn, error) {
var out []store.Turn
for _, t := range f.turns {
if t.SessionID == session && t.Owner == owner {
out = append(out, t)
}
}
return out, nil
}
func (f *fakeStore) Turn(_ context.Context, _, owner string, id uuid.UUID) (store.Turn, error) {
t, ok := f.turns[id]
if !ok || t.Owner != owner {
return store.Turn{}, store.ErrNotFound
}
return t, nil
}
func (f *fakeStore) RequestCancel(ctx context.Context, org, owner string, id uuid.UUID) (bool, error) {
_, err := f.Turn(ctx, org, owner, id)
return false, err
}
func (f *fakeStore) Events(_ context.Context, turn uuid.UUID, after int64) ([]store.Event, error) {
var out []store.Event
for _, e := range f.events[turn] {
if e.Seq > after {
out = append(out, e)
}
}
return out, nil
}
func (f *fakeStore) EventsOf(context.Context, uuid.UUID, []string) ([]store.Event, error) {
return nil, nil
}