package chat import ( "encoding/json/jsontext" "errors" "log/slog" "testing" "github.com/google/uuid" "github.com/samcharles93/archie-core/internal/platform/hub" "github.com/samcharles93/archie-core/modules/chat/internal/store" "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/governance" ) type effectsBox struct { decls []connectors.Declaration tool *countingTool } func (b effectsBox) Declarations() []connectors.Declaration { return b.decls } func (b effectsBox) Tool(string) (connectors.Tool, bool) { return b.tool, true } // TestChatOffersAndRunsOnlyDeclaredHarmlessTools guards the chat prohibition: // even for the organisation owner, a tool that changes privileges, accesses // secrets or never declared its effects is not described to the model, and a // call to it is refused before anything runs, including a call made with a // declaration that was harmless when it was offered. func TestChatOffersAndRunsOnlyDeclaredHarmlessTools(t *testing.T) { declare := func(name string, p connectors.PrivilegedEffects) connectors.Declaration { return connectors.Declaration{Connector: "iam", Kind: connectors.KindLookup, Name: name, Privileged: p} } tests := []struct { name string decl connectors.Declaration offered bool }{ {name: "declared harmless", decl: declare("who", noPrivilegedEffects), offered: true}, {name: "privilege change", decl: declare("grant", connectors.PrivilegedEffects{Declared: true, PrivilegeChange: true})}, {name: "secret access", decl: declare("export", connectors.PrivilegedEffects{Declared: true, SecretAccess: true})}, {name: "never declared", decl: declare("legacy", connectors.PrivilegedEffects{})}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { people := fakePeople{alice: person(alice, governance.RoleOwner)} box := effectsBox{decls: []connectors.Declaration{tt.decl}, tool: &countingTool{}} d := Deps{People: people, Access: governance.NewEngine(governance.ShippedOrgPolicies(governance.DefaultOrgID)), Toolbox: box} m := &Module{ orgID: "default", log: slog.New(slog.DiscardHandler), hub: hub.New(), store: &fakeStore{sessions: map[uuid.UUID]store.Session{}, turns: map[uuid.UUID]store.Turn{}, events: map[uuid.UUID][]store.Event{}}, } ctx, err := m.actingAs(t.Context(), d, alice) if err != nil { t.Fatal(err) } turn := store.Turn{ID: uuid.New(), SessionID: uuid.New(), Owner: alice} if _, ok := m.tools(ctx, d, turn)[toolName("iam", tt.decl.Name)]; ok != tt.offered { t.Fatalf("offered = %v, want %v", ok, tt.offered) } // The model can name a tool that was never offered, or one whose // declaration changed after it was offered. stale := tt.decl stale.Privileged = noPrivilegedEffects _, err = m.call(ctx, d, turn, stale, jsontext.Value(`{}`)) if tt.offered { if err != nil { t.Fatalf("call = %v, want it to run", err) } return } if !errors.Is(err, errToolProhibited) { t.Fatalf("call = %v, want %v", err, errToolProhibited) } if box.tool.lookups != 0 { t.Fatalf("a prohibited tool ran %d times", box.tool.lookups) } }) } }