Files
archie/internal/tools/guardrail_test.go
T
sam 8356863712 feat: Tools P1 — guardrails, classification, budget, availability, disclosure, dispatch
- ToolCallGuardrailConfig + GuardrailEngine with RecordSuccess/RecordFailure
- ToolClassification bitmask (idempotent/mutating) with dispatch gating
- TurnBudget for token/char accounting with spill-to-disk
- AvailabilityCache with TTL + failure grace window
- Progressive disclosure engine for tool schema filtering
- Tool dispatch executor with parallel/serial classification
- RecordSuccess fix: now returns GuardrailDecision (no-progress threshold)
- All tests pass with -race, 58 new test functions across 6 test files
2026-07-26 09:53:38 +10:00

320 lines
8.3 KiB
Go

package tools
import (
"errors"
"testing"
)
func TestDefaultGuardrailConfig(t *testing.T) {
c := DefaultGuardrailConfig()
if c.ExactFailureWarnAfter != 3 {
t.Errorf("ExactFailureWarnAfter = %d, want 3", c.ExactFailureWarnAfter)
}
if c.SameToolFailureWarnAfter != 5 {
t.Errorf("SameToolFailureWarnAfter = %d, want 5", c.SameToolFailureWarnAfter)
}
if c.NoProgressWarnAfter != 10 {
t.Errorf("NoProgressWarnAfter = %d, want 10", c.NoProgressWarnAfter)
}
if c.HardStopAfterWarnRepeat != 2 {
t.Errorf("HardStopAfterWarnRepeat = %d, want 2", c.HardStopAfterWarnRepeat)
}
}
func TestGuardrailConfigValidate(t *testing.T) {
t.Run("default config is valid", func(t *testing.T) {
if err := DefaultGuardrailConfig().Validate(); err != nil {
t.Errorf("default config should be valid: %v", err)
}
})
t.Run("zero values are valid", func(t *testing.T) {
// Zero means "never trigger" -- valid.
if err := (ToolCallGuardrailConfig{}).Validate(); err != nil {
t.Errorf("zero config should be valid: %v", err)
}
})
t.Run("negative values are invalid", func(t *testing.T) {
tests := []struct {
name string
config ToolCallGuardrailConfig
}{
{"negative ExactFailureWarnAfter", ToolCallGuardrailConfig{ExactFailureWarnAfter: -1}},
{"negative SameToolFailureWarnAfter", ToolCallGuardrailConfig{SameToolFailureWarnAfter: -1}},
{"negative NoProgressWarnAfter", ToolCallGuardrailConfig{NoProgressWarnAfter: -1}},
{"negative HardStopAfterWarnRepeat", ToolCallGuardrailConfig{HardStopAfterWarnRepeat: -1}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := tt.config.Validate(); err == nil {
t.Error("expected error for negative value")
}
})
}
})
}
// --- Guardrail decision engine tests ---
func TestGuardrailEngineAllow(t *testing.T) {
// Below all thresholds -- should allow.
g := NewGuardrailEngine(DefaultGuardrailConfig())
// A few successes -- no issues.
g.RecordSuccess("tool-a")
g.RecordSuccess("tool-b")
g.RecordSuccess("tool-a")
if g.TotalCalls() != 3 {
t.Errorf("TotalCalls = %d, want 3", g.TotalCalls())
}
}
func TestGuardrailEngineExactFailureWarn(t *testing.T) {
config := ToolCallGuardrailConfig{
ExactFailureWarnAfter: 2,
HardStopAfterWarnRepeat: 99, // effectively never hard-stop
}
g := NewGuardrailEngine(config)
err := errors.New("connection refused")
// First failure -- allow.
d := g.RecordFailure("tool-a", err)
if d != DecisionAllow {
t.Errorf("first failure should be allowed, got %v", d)
}
// Second same failure -- warn.
d = g.RecordFailure("tool-a", err)
if d != DecisionWarn {
t.Errorf("second same failure should warn, got %v", d)
}
// Different error on same tool -- doesn't trigger exact-failure warn yet.
d = g.RecordFailure("tool-a", errors.New("timeout"))
if d != DecisionAllow {
t.Errorf("different error should be allowed, got %v", d)
}
}
func TestGuardrailEngineSameToolFailureWarn(t *testing.T) {
config := ToolCallGuardrailConfig{
SameToolFailureWarnAfter: 3,
HardStopAfterWarnRepeat: 99,
}
g := NewGuardrailEngine(config)
// Three different errors on the same tool -- same-tool-failure triggers.
g.RecordFailure("tool-a", errors.New("err1"))
g.RecordFailure("tool-a", errors.New("err2"))
d := g.RecordFailure("tool-a", errors.New("err3"))
if d != DecisionWarn {
t.Errorf("third same-tool failure should warn, got %v", d)
}
// A success resets the consecutive counter.
g.RecordSuccess("tool-a")
d = g.RecordFailure("tool-a", errors.New("err4"))
if d != DecisionAllow {
t.Errorf("after success, failure should be allow, got %v", d)
}
}
func TestGuardrailEngineNoProgressWarn(t *testing.T) {
config := ToolCallGuardrailConfig{
NoProgressWarnAfter: 5,
HardStopAfterWarnRepeat: 99,
}
g := NewGuardrailEngine(config)
for range 4 {
g.RecordSuccess("tool-a")
}
// 5th call -- warn (no progress).
d := g.RecordSuccess("tool-a")
if d != DecisionWarn {
t.Errorf("5th call with no progress should warn, got %v", d)
}
}
func TestGuardrailEngineMarkProgress(t *testing.T) {
config := ToolCallGuardrailConfig{
NoProgressWarnAfter: 5,
}
g := NewGuardrailEngine(config)
for range 4 {
g.RecordSuccess("tool-a")
}
// Mark progress resets the counter.
g.MarkProgress()
d := g.RecordSuccess("tool-b")
if d != DecisionAllow {
t.Errorf("after MarkProgress, should be allow, got %v", d)
}
if g.TotalCalls() != 1 {
t.Errorf("after MarkProgress, TotalCalls = %d, want 1", g.TotalCalls())
}
}
func TestGuardrailEngineHardStop(t *testing.T) {
config := ToolCallGuardrailConfig{
ExactFailureWarnAfter: 1,
HardStopAfterWarnRepeat: 2,
}
g := NewGuardrailEngine(config)
err := errors.New("fatal")
// First failure -- warn.
d := g.RecordFailure("tool-a", err)
if d != DecisionWarn {
t.Errorf("expected warn, got %v", d)
}
// Second same failure -- hard-stop (warn repeat threshold reached).
d = g.RecordFailure("tool-a", err)
if d != DecisionHardStop {
t.Errorf("expected hard-stop, got %v", d)
}
if !g.HardStopped() {
t.Error("HardStopped should be true")
}
// All subsequent calls are hard-stop.
d = g.RecordSuccess("tool-b")
if d != DecisionHardStop {
t.Errorf("after hard-stop, all calls should be hard-stop, got %v", d)
}
}
func TestGuardrailEngineReset(t *testing.T) {
config := ToolCallGuardrailConfig{
ExactFailureWarnAfter: 1,
HardStopAfterWarnRepeat: 1,
}
g := NewGuardrailEngine(config)
g.RecordFailure("tool-a", errors.New("err"))
g.RecordFailure("tool-a", errors.New("err"))
if !g.HardStopped() {
t.Fatal("expected hard-stopped")
}
g.Reset()
if g.HardStopped() {
t.Error("after reset, should not be hard-stopped")
}
if g.TotalCalls() != 0 {
t.Errorf("after reset, TotalCalls = %d, want 0", g.TotalCalls())
}
// Can record normally again.
d := g.RecordSuccess("tool-b")
if d != DecisionAllow {
t.Errorf("after reset, should allow, got %v", d)
}
}
func TestGuardrailDecisionString(t *testing.T) {
tests := []struct {
d GuardrailDecision
want string
}{
{DecisionAllow, "allow"},
{DecisionWarn, "warn"},
{DecisionHardStop, "hard-stop"},
{GuardrailDecision(99), "unknown"},
}
for _, tt := range tests {
if got := tt.d.String(); got != tt.want {
t.Errorf("String() = %q, want %q", got, tt.want)
}
}
}
func TestGuardrailEngineConcurrent(t *testing.T) {
// Use config with all thresholds effectively disabled to avoid
// guardrail-induced short-circuits during concurrent stress.
config := ToolCallGuardrailConfig{
ExactFailureWarnAfter: 99999,
SameToolFailureWarnAfter: 99999,
NoProgressWarnAfter: 99999,
HardStopAfterWarnRepeat: 0, // never hard-stop
}
g := NewGuardrailEngine(config)
done := make(chan struct{})
for range 10 {
go func() {
for range 100 {
g.RecordSuccess("tool")
g.RecordFailure("tool", errors.New("err"))
g.TotalCalls()
g.HardStopped()
}
done <- struct{}{}
}()
}
for range 10 {
<-done
}
// Should not panic -- race detector will catch issues.
if g.TotalCalls() != 1000*2 {
t.Errorf("TotalCalls = %d, want 2000", g.TotalCalls())
}
}
func TestGuardrailEngineRecordFailureNilError(t *testing.T) {
// Regression: RecordFailure used err.Error() without a nil check,
// causing a nil-pointer dereference panic. After fix, nil error is
// handled gracefully.
g := NewGuardrailEngine(DefaultGuardrailConfig())
// Must not panic.
d := g.RecordFailure("tool-a", nil)
if d != DecisionAllow {
t.Errorf("nil error should be DecisionAllow, got %v", d)
}
if g.TotalCalls() != 1 {
t.Errorf("TotalCalls = %d, want 1 (call should still be counted)", g.TotalCalls())
}
}
func TestGuardrailEngineRecordFailureNilErrorDoesNotCountAsFailure(t *testing.T) {
// Nil error is treated as "unknown failure" and does not increment
// failure counters. Successive real failures should behave normally.
config := ToolCallGuardrailConfig{
ExactFailureWarnAfter: 2,
}
g := NewGuardrailEngine(config)
// Nil error — allowed, not counted toward exact-failure.
g.RecordFailure("tool-a", nil)
// Real error — this is the first real failure.
realErr := errors.New("real error")
d := g.RecordFailure("tool-a", realErr)
if d != DecisionAllow {
t.Errorf("first real failure should be allowed, got %v", d)
}
// Second real failure — should warn.
d = g.RecordFailure("tool-a", realErr)
if d != DecisionWarn {
t.Errorf("second real failure should warn, got %v", d)
}
}