Files
archie/tools/contractaudit/audit_test.go
T
sam 6fd76ae550 fix(tools): stop testing on the text of other files, and ungate contractaudit
Six tests asserted on the wording and formatting of source and config files
rather than on behaviour: four injected text into fixtures and checked that a
regex noticed, and two asserted on api.jsx's contents. They break on an extra
space, a newline, a rename, or a comment that merely mentions the token -- the
single-fetch check had already false-positived on a comment, and I patched it
with a comment-stripper instead of deleting it. A test that cannot tell "the
code is wrong" from "the file is formatted differently" tests the wrong thing,
and the maintenance cost is permanent.

Deleted: tools/contractaudit/negative_test.go in full, and the source-scanning
"api.jsx holds exactly one fetch call" check with its stripComments helper. The
dashboard suite still asserts the browser mutation contract behaviourally, per
method, by inspecting the request each one makes -- which is where the real
defect was found. 142 UI tests pass; the 45 methods remain fully covered by the
two inventories and the completeness check.

The tool's own decoding is now testable without a file: parseDeclarations takes
an io.Reader, so its rules are exercised with in-memory input, and only the
absent-file branch still touches a path (which is never created).

contractaudit is no longer part of task check. Its surfaces are discovered by
scanning source text, so a rename or reformat elsewhere can change its output
with nothing actually wrong -- it reported 16 live routes as unconsumed once,
when the dashboard's request helper was renamed. A check that can fail for
reasons unrelated to what it measures must not be able to break a build. It now
matches tools/reachaudit: advisory, run by hand, --strict available on request.

Also removes five unused indirect requires from tools/go.mod (go mod tidy).
They were residue from the reflection-based first implementation of the RPC
scan, which was replaced by text scanning without tidying. -mod=readonly cannot
catch a superfluous require, so nothing would ever have found them.

Rewrites docs/prds/contract-conformance-audit.md, which claimed RPC coverage was
derived by reflection over the exported client type -- it never was -- listed
surface sizes of 43/12/52 against real counts of 45/21/49, described three
classification classes where the code has four, invented a five-surface Shape
with files that were never written, and blamed raw fetch calls for a 401 defect
that did not exist. Each false claim is now recorded in an errata section rather
than quietly dropped, and the document states plainly that the audit has found
nothing actionable: the 415 refusals and the duplicated client plumbing were
both found by reading the handler and reproducing the request.

Testing rule added to the document, since this is the second time: assert on the
observable effect, never on the content or formatting of generated or
hand-authored text and config files.
2026-09-15 14:17:53 +10:00

134 lines
4.9 KiB
Go

package main
import (
"path/filepath"
"strings"
"testing"
)
// TestClassify covers the three-state rule plus the staleness rule that runs
// in the opposite direction. The staleness cases are the important ones: an
// allowlist that permits something which is no longer true is how a gate
// quietly stops checking anything.
func TestClassify(t *testing.T) {
subjects := []Subject{
{Name: "Svc/Consumed", Evidence: "proto:x", Consumed: true, Consumer: "internal/infrastructure/staterpc/client.go:41"},
{Name: "Svc/Declared", Evidence: "proto:y", Consumed: false},
{Name: "Svc/Bare", Evidence: "proto:z", Consumed: false},
{Name: "Svc/FreshlyConsumed", Evidence: "proto:w", Consumed: true, Consumer: "internal/foo/bar.go:9"},
}
declarations := Declarations{Surfaces: map[string]map[string]Declaration{
"proto/state/v1": {
"Svc/Declared": {Reason: "no consumer yet; extraction is planned", Tracker: "archie-core-8cda.4"},
"Svc/FreshlyConsumed": {Reason: "was unconsumed at audit time", Tracker: "archie-core-8cda.6"},
"Svc/Gone": {Reason: "surface removed it", Tracker: "archie-core-8cda.7"},
},
}}
findings := classify("proto/state/v1", subjects, declarations)
got := map[string]Class{}
for _, finding := range findings {
got[finding.Subject] = finding.Class
}
want := map[string]Class{
"Svc/Consumed": Consumed,
"Svc/Declared": DeclaredUnconsumed,
"Svc/Bare": Undeclared,
"Svc/FreshlyConsumed": Stale,
"Svc/Gone": Stale,
}
for subject, class := range want {
if got[subject] != class {
t.Errorf("classify(%q) = %q, want %q", subject, got[subject], class)
}
}
if len(findings) != len(want) {
t.Errorf("classify produced %d findings, want %d", len(findings), len(want))
}
}
// TestClassifyDeclaredEntryKeepsProvenance asserts the reason and tracker
// survive classification, because the declaration file is the only place the
// debt is legible.
func TestClassifyDeclaredEntryKeepsProvenance(t *testing.T) {
subjects := []Subject{{Name: "Svc/Declared"}}
declarations := Declarations{Surfaces: map[string]map[string]Declaration{
"s": {"Svc/Declared": {Reason: "the consequence, stated", Tracker: "archie-core-abc.1"}},
}}
findings := classify("s", subjects, declarations)
if len(findings) != 1 {
t.Fatalf("got %d findings, want 1", len(findings))
}
if findings[0].Evidence != "the consequence, stated" {
t.Errorf("evidence = %q, want the reason", findings[0].Evidence)
}
if findings[0].Tracker != "archie-core-abc.1" {
t.Errorf("tracker = %q, want the bead id", findings[0].Tracker)
}
}
// TestUndeclaredCountCountsOnlyFindings asserts a healthy surface contributes
// zero, so a strict run over a fully-declared surface passes.
func TestUndeclaredCountCountsOnlyFindings(t *testing.T) {
findings := []Finding{
{Subject: "a", Class: Consumed},
{Subject: "b", Class: DeclaredUnconsumed},
{Subject: "c", Class: Undeclared},
{Subject: "d", Class: Stale},
}
if got := UndeclaredCount(findings); got != 2 {
t.Errorf("UndeclaredCount = %d, want 2 (undeclared + stale)", got)
}
}
// TestLoadDeclarationsMissingFileIsEmpty asserts an absent allowlist is the
// correct starting state: every unconsumed subject becomes undeclared, which
// is the honest first report rather than a silent pass. The path is never
// created, so no file content is involved.
func TestLoadDeclarationsMissingFileIsEmpty(t *testing.T) {
declarations, err := loadDeclarations(filepath.Join(t.TempDir(), "absent.json"))
if err != nil {
t.Fatalf("loadDeclarations(absent) error = %v, want nil", err)
}
if len(declarations.Surfaces) != 0 {
t.Errorf("surfaces = %v, want empty", declarations.Surfaces)
}
}
// TestParseDeclarations covers the decoding rules in memory, so no file is
// authored and no formatting of an external file can affect the outcome.
func TestParseDeclarations(t *testing.T) {
t.Run("an empty object is usable", func(t *testing.T) {
declarations, err := parseDeclarations(strings.NewReader("{}"))
if err != nil {
t.Fatalf("error = %v, want nil", err)
}
if declarations.Surfaces == nil {
t.Error("surfaces = nil, want a non-nil empty map so a lookup cannot panic")
}
})
t.Run("entries are decoded", func(t *testing.T) {
declarations, err := parseDeclarations(strings.NewReader(
`{"surfaces":{"s":{"Svc/Method":{"reason":"because","tracker":"abc.1"}}}}`,
))
if err != nil {
t.Fatalf("error = %v, want nil", err)
}
entry := declarations.Surfaces["s"]["Svc/Method"]
if entry.Reason != "because" || entry.Tracker != "abc.1" {
t.Errorf("entry = %+v, want the reason and tracker preserved", entry)
}
})
t.Run("malformed input is rejected", func(t *testing.T) {
// A broken allowlist must fail loudly rather than degrading to "no
// declarations", which would turn every declared entry into a finding
// nobody could clear.
if _, err := parseDeclarations(strings.NewReader("{not json")); err == nil {
t.Error("error = nil, want a parse failure")
}
})
}