Files
archie/modules/judgement/chat_test.go
T
sam bc72655c8e feat(judgement): refuse privileged and undeclared actions in chat (#2071)
Connector actions and lookups now declare privilege change and secret access,
and an MCP server declares both per installation. Chat withholds and refuses
any tool that has either effect or declared nothing, in judgement and at
execution, so an approval cannot convert the refusal. Workflows and the
dashboard are unchanged.
2026-11-21 21:28:06 +11:00

110 lines
5.1 KiB
Go

package judgement
import (
"context"
"encoding/json/jsontext"
"errors"
"log/slog"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/samcharles93/archie-core/modules/connectors"
"github.com/samcharles93/archie-core/modules/judgement/internal/store"
)
func chatTestAction(p connectors.PrivilegedEffects) Action {
input := jsontext.Value(`{"role":"owner"}`)
return Action{
Connector: "iam", Name: "grant", Input: input, Digest: connectors.Digest("grant", input),
Effect: connectors.Effect{Privileged: p},
}
}
// TestChatProhibitsPrivilegedActions guards the chat prohibition: in a chat
// turn an action that changes privileges, accesses secrets or never declared
// its effects is refused at proposal and at execution, with or without an
// approval, and nothing reaches the connector. The same action in another
// workflow is judged as usual, so the dashboard and workflow runs stay open.
// The rule reads only the action and the workflow, so it holds for an owner
// as it does for anyone.
func TestChatProhibitsPrivilegedActions(t *testing.T) {
tests := []struct {
name string
workflow string
privileged connectors.PrivilegedEffects
wantRefusal bool
wantDecision Level
}{
{name: "chat refuses a privilege change", workflow: ChatWorkflow, privileged: connectors.PrivilegedEffects{Declared: true, PrivilegeChange: true}, wantRefusal: true},
{name: "chat refuses secret access", workflow: ChatWorkflow, privileged: connectors.PrivilegedEffects{Declared: true, SecretAccess: true}, wantRefusal: true},
{name: "chat refuses an action that never declared its effects", workflow: ChatWorkflow, privileged: connectors.PrivilegedEffects{}, wantRefusal: true},
{name: "chat runs an action that declared no privileged effect", workflow: ChatWorkflow, privileged: connectors.PrivilegedEffects{Declared: true}, wantDecision: ActQuietly},
{name: "a workflow run may change privileges", workflow: "provision", privileged: connectors.PrivilegedEffects{Declared: true, PrivilegeChange: true}, wantDecision: ActQuietly},
{name: "a workflow run may access secrets", workflow: "rotate", privileged: connectors.PrivilegedEffects{Declared: true, SecretAccess: true}, wantDecision: ActQuietly},
{name: "a workflow run may use an undeclared action", workflow: "provision", privileged: connectors.PrivilegedEffects{}, wantDecision: ActQuietly},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := chatTestAction(tt.privileged)
var guarded atomic.Int32
m := &Module{guards: []Guard{fixedGuard{name: "g", ceiling: ActQuietly, calls: &guarded}}}
d := m.decide(withRun(t.Context(), RunContext{Workflow: tt.workflow}), a)
if tt.wantRefusal {
if d.Level != Never || d.RefusedBy != chatProhibitionRule {
t.Fatalf("decision = level %s refused by %q, want never by %q", d.Level, d.RefusedBy, chatProhibitionRule)
}
if guarded.Load() != 0 {
t.Fatal("a guard ran for a prohibited action")
}
} else if d.Level != tt.wantDecision || d.RefusedBy != "" {
t.Fatalf("decision = level %s refused by %q, want %s", d.Level, d.RefusedBy, tt.wantDecision)
}
// Execution checks again, on a decision that reached a runnable or
// held level by any other route.
var invoked atomic.Int32
exec := &Module{tools: oneTool{countingTool{&invoked}}}
for _, level := range []Level{ActQuietly, ProposeAndWait} {
held := Decision{ID: uuid.New(), Level: level, Workflow: tt.workflow, Action: a}
approval := Approval{DecisionID: held.ID, Digest: a.Digest, By: "owner", At: time.Now().UTC()}
_, errRun := exec.Execute(t.Context(), held, a.Input, "k")
_, errApproved := exec.ExecuteApproved(t.Context(), held, approval, a.Input, "k")
if tt.wantRefusal && (!errors.Is(errRun, ErrRefused) || !errors.Is(errApproved, ErrRefused)) {
t.Fatalf("level %s: execute = %v, approved = %v; want both refused", level, errRun, errApproved)
}
}
if tt.wantRefusal && invoked.Load() != 0 {
t.Fatalf("connector invoked %d times for a prohibited action", invoked.Load())
}
if !tt.wantRefusal && invoked.Load() == 0 {
t.Fatal("an action outside the prohibition never ran")
}
})
}
}
type failingPolicies struct{ memoryPolicies }
func (failingPolicies) Policy(context.Context, string, string, int) (store.PolicySet, error) {
return store.PolicySet{}, errors.New("database unavailable")
}
// TestUnreadableHardRulesRefuse guards fail-closed judgement: when the hard
// rules can't be read, no action is judged to run, in chat or anywhere else.
func TestUnreadableHardRulesRefuse(t *testing.T) {
for _, workflow := range []string{ChatWorkflow, "provision"} {
t.Run(workflow, func(t *testing.T) {
p := &Policies{store: &failingPolicies{}, log: slog.New(slog.DiscardHandler)}
m := &Module{rules: []HardRule{p}, guards: []Guard{fixedGuard{name: "g", ceiling: ActQuietly}}}
a := chatTestAction(connectors.PrivilegedEffects{Declared: true})
d := m.decide(withRun(t.Context(), RunContext{Workflow: workflow}), a)
if d.Level != Never || d.RefusedBy != hardRulesUnreadable {
t.Fatalf("decision = level %s refused by %q, want never by %q", d.Level, d.RefusedBy, hardRulesUnreadable)
}
})
}
}