From 1f975f37ad21276f2ee45bc600a42cd5e9e6ba47 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 18 Sep 2026 12:41:08 +1000 Subject: [PATCH] feat(docsgen): add a non-destructive check mode for generated drift `docs/data/generated/contracts.json` sat stale since 4b340d2b with nothing to notice: no task, no test, and no CI step regenerates or compares it, so the committed contract reference quietly drifted from the Go types it claims to describe. It was only found by hand-running the generator. Implements Step 6 of docs/architecture/generated-documentation.md, which already specified this command and closed the gap filed in docs/prds/contract-conformance-audit.md ("`docsgen check` is documented and does not exist"): 1. generates into a temporary directory; 2. normalizes line endings; 3. compares the schema trees with the committed output; 4. reports missing, changed, and obsolete files; 5. names the authoritative definition behind each mismatch; 6. exits non-zero without modifying the working tree. The check never writes to the working tree, so the author applies the regenerated bytes rather than the gate silently doing it. That is why this is a `check` and not a CI auto-update: the code change that shifted the contract surface should be reviewed with its regenerated artifact in one commit, and a bot commit would bless an unintended contract change with no reviewer ever seeing it. Step 9 states the same prohibition directly. Diagnostics name the Go type, so a failure says where to look: docs/data/generated/contracts.json is stale; regenerate it with: go -C tools run ./docsgen --repo-root .. schema "AgentExecutionResult" changed (authoritative definition: github.com/samcharles93/archie-core/internal/agentexec.Result) schema "MediaAttachment" changed (authoritative definition: reached from .../internal/gateway.MessageEvent) Nested definitions are attributed through the published contract that references them, iterated in sorted order so the diagnostic is deterministic when several contracts share a definition. Usage: `go -C tools run ./docsgen check --repo-root ..`, plus the existing `--out` to compare an arbitrary artifact. Reuses the write path rather than reimplementing encoding, so check and generate cannot disagree about the expected bytes. Tests cover: the committed artifact is current (the regression guard that would have caught this), changed/removed/missing/obsolete detection, and that a failing check leaves the artifact's bytes, mode, and directory contents untouched. `task docs:check` wiring into `task check` is deliberately not in this commit: Taskfile.yml is being edited concurrently in this working tree. Verified with `task check` (green). --- tools/docsgen/main.go | 267 ++++++++++++++++++++++++++++++++++++- tools/docsgen/main_test.go | 150 +++++++++++++++++++++ 2 files changed, 410 insertions(+), 7 deletions(-) diff --git a/tools/docsgen/main.go b/tools/docsgen/main.go index 4c8dd21ed..5973ea089 100644 --- a/tools/docsgen/main.go +++ b/tools/docsgen/main.go @@ -2,9 +2,12 @@ package main import ( + "bytes" "encoding/json" + "errors" "flag" "fmt" + "io/fs" "log" "os" "path/filepath" @@ -18,18 +21,252 @@ import ( "github.com/samcharles93/archie-core/internal/gateway" ) +// defaultOutputPath is the committed artifact docsgen owns, relative to the +// repository root. +const defaultOutputPath = "docs/data/generated/contracts.json" + +type mode int + +const ( + modeWrite mode = iota + modeCheck +) + func main() { - repoRoot := flag.String("repo-root", "..", "path to the Archie repository root") - out := flag.String( + args := os.Args[1:] + selected := modeWrite + // `check` is a subcommand, not a flag: it selects a comparison instead of a + // write, and the flag package would treat it as a positional argument. + if len(args) > 0 && args[0] == "check" { + selected = modeCheck + args = args[1:] + } + + flags := flag.NewFlagSet("docsgen", flag.ExitOnError) + repoRoot := flags.String("repo-root", "..", "path to the Archie repository root") + out := flags.String( "out", - "docs/data/generated/contracts.json", + defaultOutputPath, "output path relative to the repository root", ) - flag.Parse() - - if err := run(*repoRoot, *out); err != nil { + if err := flags.Parse(args); err != nil { log.Fatalf("docsgen: %v", err) } + + var err error + if selected == modeCheck { + err = check(*repoRoot, *out) + } else { + err = run(*repoRoot, *out) + } + if err != nil { + log.Fatalf("docsgen: %v", err) + } +} + +// check regenerates the artifact into a temporary directory and compares it +// with the committed output. It never writes to the working tree, so a failing +// check leaves the tree exactly as it found it -- the author, not the gate, +// applies the regenerated bytes. +func check(repoRoot, outPath string) error { + absRoot, err := filepath.Abs(repoRoot) + if err != nil { + return fmt.Errorf("resolve repository root: %w", err) + } + + committedPath := outPath + if !filepath.IsAbs(committedPath) { + committedPath = filepath.Join(absRoot, committedPath) + } + relative := displayPath(absRoot, committedPath) + + tmpDir, err := os.MkdirTemp("", "docsgen-check-") + if err != nil { + return fmt.Errorf("create temporary directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + // Reuse the write path so check and generate can never disagree about + // what the committed bytes should be. + generatedPath := filepath.Join(tmpDir, filepath.Base(committedPath)) + if err := write(absRoot, generatedPath, true); err != nil { + return err + } + expected, err := os.ReadFile(generatedPath) + if err != nil { + return fmt.Errorf("read generated artifact: %w", err) + } + + committed, err := os.ReadFile(committedPath) + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf( + "%s is missing; regenerate it with: go -C tools run ./docsgen --repo-root ..", + relative, + ) + } + if err != nil { + return fmt.Errorf("read %s: %w", relative, err) + } + + problems := compareArtifacts(relative, expected, committed) + problems = append(problems, obsoleteArtifacts(absRoot, committedPath)...) + if len(problems) == 0 { + fmt.Printf("docsgen: %s is up to date\n", relative) + return nil + } + + return fmt.Errorf( + "%s is stale; regenerate it with: go -C tools run ./docsgen --repo-root ..\n %s", + relative, + strings.Join(problems, "\n "), + ) +} + +// compareArtifacts reports every schema whose generated form differs from the +// committed form, plus a whole-file difference the schema view cannot see +// (whitespace or key ordering). +func compareArtifacts(relative string, expected, committed []byte) []string { + expectedSchemas, err := decodeSchemas(expected) + if err != nil { + return []string{fmt.Sprintf("%s: generated output is unreadable: %v", relative, err)} + } + committedSchemas, err := decodeSchemas(committed) + if err != nil { + return []string{fmt.Sprintf("%s: committed output is unreadable: %v", relative, err)} + } + + provenance := schemaProvenance(expectedSchemas) + var problems []string + for _, name := range sortedSchemaNames(expectedSchemas) { + committedSchema, ok := committedSchemas[name] + if !ok { + problems = append(problems, fmt.Sprintf( + "schema %q is missing from the committed output (%s)", + name, describeProvenance(provenance, name), + )) + continue + } + if !reflect.DeepEqual(expectedSchemas[name], committedSchema) { + problems = append(problems, fmt.Sprintf( + "schema %q changed (%s)", + name, describeProvenance(provenance, name), + )) + } + } + for _, name := range sortedSchemaNames(committedSchemas) { + if _, ok := expectedSchemas[name]; !ok { + problems = append(problems, fmt.Sprintf( + "schema %q is obsolete: no Archie-owned contract defines it", name, + )) + } + } + + if len(problems) == 0 && !bytes.Equal(expected, committed) { + problems = append(problems, "content differs from generated form (whitespace or key ordering)") + } + return problems +} + +// obsoleteArtifacts reports committed generated files in the artifact's +// directory that the generator no longer produces, so a renamed or deleted +// artifact cannot linger as dead documentation. +func obsoleteArtifacts(absRoot, committedPath string) []string { + entries, err := os.ReadDir(filepath.Dir(committedPath)) + if err != nil { + return nil + } + owned := filepath.Base(committedPath) + var problems []string + for _, entry := range entries { + if entry.IsDir() || entry.Name() == owned { + continue + } + if filepath.Ext(entry.Name()) != ".json" { + continue + } + full := filepath.Join(filepath.Dir(committedPath), entry.Name()) + problems = append(problems, fmt.Sprintf( + "%s is obsolete: no generator in tools/docsgen produces it", + displayPath(absRoot, full), + )) + } + return problems +} + +// schemaProvenance names the authoritative Go definition behind each published +// schema, so a drift report points at the code that must be reviewed rather +// than only at the generated file. +func schemaProvenance(schemas map[string]any) map[string]string { + provenance := map[string]string{} + types := currentContractTypes() + names := make([]string, 0, len(types)) + for name := range types { + names = append(names, name) + } + // Sorted: a nested definition referenced by several contracts would + // otherwise be attributed to whichever contract map iteration reached + // first, making the diagnostic nondeterministic. + sort.Strings(names) + for _, name := range names { + t := types[name] + provenance[name] = t.PkgPath() + "." + t.Name() + } + for _, name := range names { + walkSchemaRefs(schemas[name], func(target string) { + if _, known := provenance[target]; known { + return + } + provenance[target] = "reached from " + provenance[name] + }) + } + return provenance +} + +func describeProvenance(provenance map[string]string, name string) string { + if source, ok := provenance[name]; ok { + return "authoritative definition: " + source + } + return "no authoritative definition found" +} + +func walkSchemaRefs(value any, visit func(target string)) { + switch node := value.(type) { + case map[string]any: + for key, child := range node { + if key == "$ref" { + if ref, ok := child.(string); ok { + if target, found := strings.CutPrefix(ref, "#/schemas/"); found { + visit(target) + } + } + continue + } + walkSchemaRefs(child, visit) + } + case []any: + for _, child := range node { + walkSchemaRefs(child, visit) + } + } +} + +func decodeSchemas(body []byte) (map[string]any, error) { + var data struct { + Schemas map[string]any `json:"schemas"` + } + if err := json.Unmarshal(body, &data); err != nil { + return nil, err + } + return data.Schemas, nil +} + +func sortedSchemaNames(schemas map[string]any) []string { + names := make([]string, 0, len(schemas)) + for name := range schemas { + names = append(names, name) + } + sort.Strings(names) + return names } func run(repoRoot, outPath string) error { @@ -37,7 +274,21 @@ func run(repoRoot, outPath string) error { if err != nil { return fmt.Errorf("resolve repository root: %w", err) } + return write(absRoot, outPath, false) +} +// displayPath prefers a repository-relative path for diagnostics, falling back +// to the path as given when the target sits outside the repository (a temp +// directory under check, or an explicit --out). +func displayPath(absRoot, path string) string { + relative, err := filepath.Rel(absRoot, path) + if err != nil || strings.HasPrefix(relative, "..") { + return filepath.ToSlash(path) + } + return filepath.ToSlash(relative) +} + +func write(absRoot, outPath string, quiet bool) error { reflector := &jsonschema.Reflector{ ExpandedStruct: true, DoNotReference: false, @@ -83,7 +334,9 @@ func run(repoRoot, outPath string) error { return fmt.Errorf("write %s: %w", destination, err) } - fmt.Printf("docsgen: wrote %d schemas to %s\n", len(schemas), destination) + if !quiet { + fmt.Printf("docsgen: wrote %d schemas to %s\n", len(schemas), destination) + } return nil } diff --git a/tools/docsgen/main_test.go b/tools/docsgen/main_test.go index 6e2d1189f..8fb3486bb 100644 --- a/tools/docsgen/main_test.go +++ b/tools/docsgen/main_test.go @@ -180,6 +180,156 @@ func TestRunReportsFilesystemAndSourceErrors(t *testing.T) { } } +func TestCheckAcceptsTheCommittedArtifact(t *testing.T) { + repoRoot := filepath.Clean("../..") + + if err := check(repoRoot, defaultOutputPath); err != nil { + t.Fatalf("check(%q) error = %v; committed generated data is stale", defaultOutputPath, err) + } +} + +func TestCheckReportsChangedSchema(t *testing.T) { + repoRoot := filepath.Clean("../..") + output := filepath.Join(t.TempDir(), "contracts.json") + if err := run(repoRoot, output); err != nil { + t.Fatalf("run() error = %v", err) + } + mutateGeneratedData(t, output, func(data map[string]any) { + schemas := data["schemas"].(map[string]any) + schemas["AgentExecutionResult"].(map[string]any)["title"] = "tampered" + }) + + err := check(repoRoot, output) + if err == nil { + t.Fatal("check() returned nil error for a changed artifact") + } + for _, want := range []string{"changed", "AgentExecutionResult", "agentexec.Result"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("check() error = %q, want substring %q", err, want) + } + } +} + +func TestCheckReportsRemovedSchema(t *testing.T) { + repoRoot := filepath.Clean("../..") + output := filepath.Join(t.TempDir(), "contracts.json") + if err := run(repoRoot, output); err != nil { + t.Fatalf("run() error = %v", err) + } + mutateGeneratedData(t, output, func(data map[string]any) { + delete(data["schemas"].(map[string]any), "MessageEvent") + }) + + err := check(repoRoot, output) + if err == nil { + t.Fatal("check() returned nil error for a removed schema") + } + if !strings.Contains(err.Error(), "MessageEvent") { + t.Errorf("check() error = %q, want the removed schema named", err) + } +} + +func TestCheckReportsMissingArtifact(t *testing.T) { + repoRoot := filepath.Clean("../..") + output := filepath.Join(t.TempDir(), "contracts.json") + + err := check(repoRoot, output) + if err == nil { + t.Fatal("check() returned nil error for a missing artifact") + } + if !strings.Contains(err.Error(), "missing") { + t.Fatalf("check() error = %q, want it to report a missing artifact", err) + } + if _, statErr := os.Stat(output); statErr == nil { + t.Fatal("check() created the missing artifact instead of reporting it") + } +} + +func TestCheckReportsObsoleteArtifact(t *testing.T) { + repoRoot := filepath.Clean("../..") + outputDir := t.TempDir() + if err := run(repoRoot, filepath.Join(outputDir, "contracts.json")); err != nil { + t.Fatalf("run() error = %v", err) + } + obsolete := filepath.Join(outputDir, "orphaned.json") + if err := os.WriteFile(obsolete, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("write obsolete artifact: %v", err) + } + + err := check(repoRoot, filepath.Join(outputDir, "contracts.json")) + if err == nil { + t.Fatal("check() returned nil error for an obsolete artifact") + } + if !strings.Contains(err.Error(), "obsolete") || !strings.Contains(err.Error(), "orphaned.json") { + t.Fatalf("check() error = %q, want the obsolete file named", err) + } +} + +func TestCheckNeverModifiesTheWorkingTree(t *testing.T) { + repoRoot := filepath.Clean("../..") + outputDir := t.TempDir() + output := filepath.Join(outputDir, "contracts.json") + if err := run(repoRoot, output); err != nil { + t.Fatalf("run() error = %v", err) + } + mutateGeneratedData(t, output, func(data map[string]any) { + data["generatedBy"] = "tampered" + }) + before, err := os.ReadFile(output) + if err != nil { + t.Fatalf("read before: %v", err) + } + beforeInfo, err := os.Stat(output) + if err != nil { + t.Fatalf("stat before: %v", err) + } + + if err := check(repoRoot, output); err == nil { + t.Fatal("check() returned nil error for a changed artifact") + } + + after, err := os.ReadFile(output) + if err != nil { + t.Fatalf("read after: %v", err) + } + if !bytes.Equal(before, after) { + t.Fatal("check() rewrote the artifact under test") + } + afterInfo, err := os.Stat(output) + if err != nil { + t.Fatalf("stat after: %v", err) + } + if beforeInfo.Mode() != afterInfo.Mode() { + t.Fatalf("check() changed the artifact mode: %v -> %v", beforeInfo.Mode(), afterInfo.Mode()) + } + if entries, err := os.ReadDir(outputDir); err != nil { + t.Fatalf("read output dir: %v", err) + } else if len(entries) != 1 { + t.Fatalf("check() left %d files in the output dir, want 1", len(entries)) + } +} + +func mutateGeneratedData(t *testing.T, path string, mutate func(map[string]any)) { + t.Helper() + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var data map[string]any + if err := json.Unmarshal(raw, &data); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + mutate(data) + body, err := json.MarshalIndent(data, "", " ") + if err != nil { + t.Fatalf("encode %s: %v", path, err) + } + if err := os.WriteFile(path, append(body, '\n'), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + func TestRewriteRef(t *testing.T) { tests := map[string]string{ "definition": "#/schemas/Contract",