84 lines
3.0 KiB
Go
84 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archipelago/sdk/guard"
|
|
)
|
|
|
|
// TestEveryVerdictKeepsItsJudgeRecord guards the audit rule that a guard never
|
|
// discards the judge's output: a clean answer, a violation and a failed call
|
|
// each carry a record of the model, the answers and the policy it was judged on.
|
|
func TestEveryVerdictKeepsItsJudgeRecord(t *testing.T) {
|
|
const rules = "never delete production data"
|
|
tests := []struct {
|
|
name string
|
|
status int
|
|
body string
|
|
wantAnswer string
|
|
wantCeil guard.Ceiling
|
|
wantFailed bool
|
|
}{
|
|
{
|
|
name: "a complying action keeps its answer", status: http.StatusOK, wantCeil: guard.ActQuietly, wantAnswer: "complies",
|
|
body: `{"id":"r1","model":"m","answers":{"verdict":{"type":"choice","choice":"complies","confidence":0.97},"rule":{"type":"choice","choice":"none"}}}`,
|
|
},
|
|
{
|
|
name: "a violation keeps its answer", status: http.StatusOK, wantCeil: guard.ProposeAndWait, wantAnswer: "violates",
|
|
body: `{"id":"r2","model":"m","answers":{"verdict":{"type":"choice","choice":"violates","confidence":0.95},"rule":{"type":"choice","choice":"rule 1"}}}`,
|
|
},
|
|
{name: "a failed call keeps the error", status: http.StatusBadGateway, body: "down", wantCeil: guard.ProposeAndWait, wantFailed: true},
|
|
}
|
|
sum := sha256.Sum256([]byte(rules))
|
|
digest := hex.EncodeToString(sum[:])
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(tt.status)
|
|
_, _ = w.Write([]byte(tt.body))
|
|
}))
|
|
defer srv.Close()
|
|
g, err := newPolicy(guard.Config{Settings: map[string]string{
|
|
"policy": rules, "judge_base_url": srv.URL, "judge_model": "m",
|
|
}}, recordedEval, time.Now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
v, err := g.Judge(t.Context(), guard.Action{Connector: "c", Name: "n"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if v.Ceiling != tt.wantCeil {
|
|
t.Errorf("ceiling = %d, want %d", v.Ceiling, tt.wantCeil)
|
|
}
|
|
if len(v.Judges) != 1 {
|
|
t.Fatalf("judges = %d, want 1", len(v.Judges))
|
|
}
|
|
j := v.Judges[0]
|
|
if j.Model != "m" || j.Basis != rules || j.BasisDigest != digest {
|
|
t.Errorf("record = %+v, want model m and the policy as basis", j)
|
|
}
|
|
if tt.wantFailed {
|
|
if j.Reason == "" || j.Answer != "" {
|
|
t.Errorf("failed record = %+v, want the error as reason", j)
|
|
}
|
|
// A failed call still records exactly what was sent and what
|
|
// came back, so the record can be audited without repeating it.
|
|
if len(j.Request) == 0 || string(j.Response) != tt.body ||
|
|
j.HTTPStatus != tt.status || j.EndpointPath == "" {
|
|
t.Errorf("failed record = %+v, want request, response %q, status %d and path", j, tt.body, tt.status)
|
|
}
|
|
return
|
|
}
|
|
if j.Answer != tt.wantAnswer || j.Confidence == 0 || len(j.Answers) == 0 || j.ResultID == "" {
|
|
t.Errorf("record = %+v, want answer %q with confidence, result id and answers", j, tt.wantAnswer)
|
|
}
|
|
})
|
|
}
|
|
}
|