Files
archipelago/guards/policy/main_test.go
T

84 lines
3.0 KiB
Go

package main
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/samcharles93/archipelago/sdk/guard"
)
// TestEveryVerdictKeepsItsJudgeRecord guards the audit rule that a guard never
// discards the judge's output: a clean answer, a violation and a failed call
// each carry a record of the model, the answers and the policy it was judged on.
func TestEveryVerdictKeepsItsJudgeRecord(t *testing.T) {
const rules = "never delete production data"
tests := []struct {
name string
status int
body string
wantAnswer string
wantCeil guard.Ceiling
wantFailed bool
}{
{
name: "a complying action keeps its answer", status: http.StatusOK, wantCeil: guard.ActQuietly, wantAnswer: "complies",
body: `{"id":"r1","model":"m","answers":{"verdict":{"type":"choice","choice":"complies","confidence":0.97},"rule":{"type":"choice","choice":"none"}}}`,
},
{
name: "a violation keeps its answer", status: http.StatusOK, wantCeil: guard.ProposeAndWait, wantAnswer: "violates",
body: `{"id":"r2","model":"m","answers":{"verdict":{"type":"choice","choice":"violates","confidence":0.95},"rule":{"type":"choice","choice":"rule 1"}}}`,
},
{name: "a failed call keeps the error", status: http.StatusBadGateway, body: "down", wantCeil: guard.ProposeAndWait, wantFailed: true},
}
sum := sha256.Sum256([]byte(rules))
digest := hex.EncodeToString(sum[:])
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(tt.status)
_, _ = w.Write([]byte(tt.body))
}))
defer srv.Close()
g, err := newPolicy(guard.Config{Settings: map[string]string{
"policy": rules, "judge_base_url": srv.URL, "judge_model": "m",
}}, recordedEval, time.Now)
if err != nil {
t.Fatal(err)
}
v, err := g.Judge(t.Context(), guard.Action{Connector: "c", Name: "n"})
if err != nil {
t.Fatal(err)
}
if v.Ceiling != tt.wantCeil {
t.Errorf("ceiling = %d, want %d", v.Ceiling, tt.wantCeil)
}
if len(v.Judges) != 1 {
t.Fatalf("judges = %d, want 1", len(v.Judges))
}
j := v.Judges[0]
if j.Model != "m" || j.Basis != rules || j.BasisDigest != digest {
t.Errorf("record = %+v, want model m and the policy as basis", j)
}
if tt.wantFailed {
if j.Reason == "" || j.Answer != "" {
t.Errorf("failed record = %+v, want the error as reason", j)
}
// A failed call still records exactly what was sent and what
// came back, so the record can be audited without repeating it.
if len(j.Request) == 0 || string(j.Response) != tt.body ||
j.HTTPStatus != tt.status || j.EndpointPath == "" {
t.Errorf("failed record = %+v, want request, response %q, status %d and path", j, tt.body, tt.status)
}
return
}
if j.Answer != tt.wantAnswer || j.Confidence == 0 || len(j.Answers) == 0 || j.ResultID == "" {
t.Errorf("record = %+v, want answer %q with confidence, result id and answers", j, tt.wantAnswer)
}
})
}
}