From 0a47a95cc33a314eb168e9e8fc5bf4b6aa2f17b4 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Thu, 8 Oct 2026 22:30:12 +1100 Subject: [PATCH] fix(agentexec): enforce capture tool result schemas --- go.mod | 2 +- internal/agentexec/helpers.go | 45 ++++++++++ internal/agentexec/helpers_test.go | 98 ++++++++++++++++++++++ internal/domain/workflow/agent_run.go | 11 +++ internal/domain/workflow/agent_run_test.go | 76 +++++++++++++++++ internal/domain/workflow/outputs.go | 6 +- internal/domain/workflow/outputs_test.go | 43 ++++++++++ 7 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 internal/agentexec/helpers_test.go create mode 100644 internal/domain/workflow/agent_run_test.go create mode 100644 internal/domain/workflow/outputs_test.go diff --git a/go.mod b/go.mod index 9a7b40d49..853c66265 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,6 @@ require ( github.com/go-ole/go-ole v1.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/google/jsonschema-go v0.4.3 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -100,6 +99,7 @@ require ( github.com/go-git/go-billy/v6 v6.0.0-alpha.2 github.com/go-git/go-git/v6 v6.0.0-alpha.5 github.com/go-telegram/bot v1.27.0 + github.com/google/jsonschema-go v0.4.3 github.com/google/uuid v1.6.0 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.8.0 diff --git a/internal/agentexec/helpers.go b/internal/agentexec/helpers.go index 91cb52f3c..fe21d431b 100644 --- a/internal/agentexec/helpers.go +++ b/internal/agentexec/helpers.go @@ -8,6 +8,8 @@ import ( "slices" "strings" + "github.com/google/jsonschema-go/jsonschema" + "github.com/samcharles93/archie-core/internal/domain/agentrun" ) @@ -74,9 +76,52 @@ func ValidateCaptureArgs(spec agentrun.CaptureTool, value json.RawMessage) (stri return rejection, false } } + // The derived checks above cover the common cases with friendly wording; + // the schema is what enforces the rest of what the tool advertised -- + // enums, numeric types, nested objects, array shapes -- so a call the + // agent could only make valid is rejected with the reason, not recorded. + return checkSchema(spec, value) +} + +// checkSchema validates the arguments against the tool's advertised JSON +// Schema. A tool with no schema, or one that will not resolve, is accepted +// here: a schema that never resolves is a definition bug, refused where the +// tool is built (workflow's agentResultTool), so the agent is never blamed +// for it. +func checkSchema(spec agentrun.CaptureTool, value json.RawMessage) (string, bool) { + if len(spec.Parameters) == 0 { + return "", true + } + resolved, err := resolveCaptureSchema(spec.Parameters) + if err != nil { + return "", true + } + var instance any + if err := json.Unmarshal(value, &instance); err != nil { + return spec.Name + " rejected: arguments must be JSON", false + } + if err := resolved.Validate(instance); err != nil { + return fmt.Sprintf("%s rejected: %v", spec.Name, err), false + } return "", true } +// resolveCaptureSchema parses and resolves a capture tool's JSON Schema. The +// workflow parser resolves the same schema when it builds the tool, so an +// unusable schema is refused at definition time rather than silently skipped +// here. +func resolveCaptureSchema(params json.RawMessage) (*jsonschema.Resolved, error) { + var schema jsonschema.Schema + if err := json.Unmarshal(params, &schema); err != nil { + return nil, fmt.Errorf("capture tool schema: %w", err) + } + resolved, err := schema.Resolve(nil) + if err != nil { + return nil, fmt.Errorf("capture tool schema: %w", err) + } + return resolved, nil +} + func checkRequiredFields(spec agentrun.CaptureTool, object map[string]json.RawMessage) (string, bool) { for _, field := range spec.RequiredFields { if _, ok := object[field]; !ok { diff --git a/internal/agentexec/helpers_test.go b/internal/agentexec/helpers_test.go new file mode 100644 index 000000000..a9031b253 --- /dev/null +++ b/internal/agentexec/helpers_test.go @@ -0,0 +1,98 @@ +package agentexec + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/domain/agentrun" +) + +// resultSchema is the shape of a shipped planner result (triage's), with a +// nested object and an array so every constraint kind the agent can violate +// is exercised. +const resultSchema = `{ + "type": "object", + "properties": { + "needs_code_change": {"type": "boolean"}, + "workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"]}, + "reasons": {"type": "string"}, + "labels": {"type": "array", "items": {"type": "string"}, "maxItems": 2}, + "plan": {"type": "object", "properties": {"risk": {"type": "integer"}}, "required": ["risk"]} + }, + "required": ["needs_code_change", "workflow", "reasons"] +}` + +// TestValidateCaptureArgsEnforcesSchema pins that the advertised schema is +// enforced, not merely offered to the agent: an enum, a numeric type, an +// array's item type and length, and a nested object's shape all reject with a +// message naming what was wrong, so the agent can correct it. +func TestValidateCaptureArgsEnforcesSchema(t *testing.T) { + base := `"needs_code_change": true, "workflow": "tdd", "reasons": "why"` + tests := []struct { + name string + args string + wantOK bool + wantText string + }{ + {name: "valid", args: `{` + base + `}`, wantOK: true}, + {name: "enum violation", args: `{` + base + `, "workflow": "banana"}`, wantText: "workflow"}, + {name: "wrong boolean type", args: `{"needs_code_change": "yes", "workflow": "tdd", "reasons": "why"}`, wantText: "needs_code_change"}, + {name: "string where integer expected", args: `{` + base + `, "plan": {"risk": "high"}}`, wantText: "plan"}, + {name: "array item type", args: `{` + base + `, "labels": [1, 2]}`, wantText: "labels"}, + {name: "array too long", args: `{` + base + `, "labels": ["a", "b", "c"]}`, wantText: "labels"}, + {name: "nested required missing", args: `{` + base + `, "plan": {}}`, wantText: "plan"}, + } + spec := agentrun.CaptureTool{Name: "result", Parameters: json.RawMessage(resultSchema)} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rejection, ok := ValidateCaptureArgs(spec, json.RawMessage(tt.args)) + if ok != tt.wantOK { + t.Fatalf("ok = %v, rejection = %q; want ok %v", ok, rejection, tt.wantOK) + } + if ok { + return + } + if !strings.HasPrefix(rejection, "result rejected: ") { + t.Fatalf("rejection %q does not name the tool", rejection) + } + if !strings.Contains(rejection, tt.wantText) { + t.Fatalf("rejection %q does not mention %q", rejection, tt.wantText) + } + }) + } +} + +// TestValidateCaptureArgsWithoutSchema pins that a tool with no schema, or one +// whose schema cannot resolve, still accepts a call: the derived constraint +// lists remain the whole check, so the agent is never blocked by a tool +// definition the schema check cannot read. +func TestValidateCaptureArgsWithoutSchema(t *testing.T) { + tests := []struct { + name string + spec agentrun.CaptureTool + args string + wantOK bool + }{ + {name: "no schema", spec: agentrun.CaptureTool{Name: "note"}, args: `{"anything": 1}`, wantOK: true}, + { + name: "unresolvable schema is not the agent's problem", + spec: agentrun.CaptureTool{Name: "note", Parameters: json.RawMessage(`{"$ref": "#/definitions/missing"}`)}, + args: `{"anything": 1}`, + wantOK: true, + }, + { + name: "derived checks still apply without a schema", + spec: agentrun.CaptureTool{Name: "note", NonEmptyStrings: []string{"body"}}, + args: `{"body": " "}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, ok := ValidateCaptureArgs(tt.spec, json.RawMessage(tt.args)) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + }) + } +} diff --git a/internal/domain/workflow/agent_run.go b/internal/domain/workflow/agent_run.go index 6b976c960..f2efcfa40 100644 --- a/internal/domain/workflow/agent_run.go +++ b/internal/domain/workflow/agent_run.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/google/jsonschema-go/jsonschema" "gopkg.in/yaml.v3" "github.com/samcharles93/archie-core/internal/domain/agentrun" @@ -143,6 +144,16 @@ func agentResultTool(schema map[string]any) (*agentrun.CaptureTool, error) { if err != nil { return nil, err } + // A schema that will not resolve cannot be enforced at call time, so the + // definition is refused here rather than accepting results it never + // checked. + var resolved jsonschema.Schema + if err := json.Unmarshal(params, &resolved); err != nil { + return nil, fmt.Errorf("must be a valid JSON Schema: %w", err) + } + if _, err := resolved.Resolve(nil); err != nil { + return nil, fmt.Errorf("must be a valid JSON Schema: %w", err) + } var required []string if list, ok := schema["required"].([]any); ok { for _, field := range list { diff --git a/internal/domain/workflow/agent_run_test.go b/internal/domain/workflow/agent_run_test.go new file mode 100644 index 000000000..d9dedbda8 --- /dev/null +++ b/internal/domain/workflow/agent_run_test.go @@ -0,0 +1,76 @@ +package workflow + +import ( + "slices" + "strings" + "testing" +) + +// TestAgentResultToolSchema covers how an agent.run step's result schema +// becomes a capture tool: none stays none, a good schema keeps its derived +// constraints, and a schema that would not resolve is refused at definition +// time rather than producing a tool the run can never satisfy. +func TestAgentResultToolSchema(t *testing.T) { + valid := map[string]any{ + "type": "object", + "properties": map[string]any{ + "needs_change": map[string]any{"type": "boolean"}, + "workflow": map[string]any{"type": "string", "enum": []any{"implement", "tdd"}}, + "reasons": map[string]any{"type": "string"}, + }, + "required": []any{"needs_change", "workflow", "reasons"}, + } + tests := []struct { + name string + schema map[string]any + wantErr string + }{ + {name: "no schema", schema: nil}, + {name: "valid schema", schema: valid}, + {name: "not an object", schema: map[string]any{"type": "string"}, wantErr: "must be a JSON Schema with type: object"}, + { + name: "unresolvable reference", + schema: map[string]any{ + "type": "object", + "properties": map[string]any{"x": map[string]any{"$ref": "#/definitions/missing"}}, + }, + wantErr: "must be a valid JSON Schema", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tool, err := agentResultTool(tt.schema) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.schema == nil { + if tool != nil { + t.Fatalf("tool = %#v, want nil for no schema", tool) + } + return + } + if tool == nil { + t.Fatal("tool = nil, want a capture tool") + } + if len(tool.RequiredFields) != 3 { + t.Fatalf("RequiredFields = %v, want the schema's three required fields", tool.RequiredFields) + } + if !containsString(tool.NonEmptyStrings, "reasons") && !containsString(tool.NonEmptyStrings, "workflow") { + t.Fatalf("NonEmptyStrings = %v, want the required string fields", tool.NonEmptyStrings) + } + if !containsString(tool.BooleanFields, "needs_change") { + t.Fatalf("BooleanFields = %v, want needs_change", tool.BooleanFields) + } + }) + } +} + +func containsString(list []string, want string) bool { + return slices.Contains(list, want) +} diff --git a/internal/domain/workflow/outputs.go b/internal/domain/workflow/outputs.go index 7597df534..a43837063 100644 --- a/internal/domain/workflow/outputs.go +++ b/internal/domain/workflow/outputs.go @@ -67,7 +67,11 @@ func (tc *TaskContext) WriteOutput(name string, value any) error { func outputCaptureTool(name string, spec task.OutputSpec) agentrun.CaptureTool { value := map[string]any{"description": fmt.Sprintf("The value of the %q output.", name)} if typ := jsonSchemaType(spec.Type); typ != "" { - value["type"] = typ + // Null is how the agent withdraws a value it cannot supply, and the + // run reads a null value as "not written" (decodeCapturedOutput), so + // the advertised schema has to permit it or the call would be + // rejected for the value the contract assigns a meaning to. + value["type"] = []string{typ, "null"} } params, err := json.Marshal(map[string]any{ "type": "object", diff --git a/internal/domain/workflow/outputs_test.go b/internal/domain/workflow/outputs_test.go new file mode 100644 index 000000000..5dde2499b --- /dev/null +++ b/internal/domain/workflow/outputs_test.go @@ -0,0 +1,43 @@ +package workflow + +import ( + "encoding/json" + "testing" + + "github.com/google/jsonschema-go/jsonschema" + + "github.com/samcharles93/archie-core/internal/domain/workflow/task" +) + +// TestOutputCaptureToolSchemaAllowsNull pins the advertised schema of a +// declared output against the contract the run applies: a typed value, or a +// null that means "not written" (decodeCapturedOutput). Enforcing the schema +// must not reject the null the contract gives a meaning to. +func TestOutputCaptureToolSchemaAllowsNull(t *testing.T) { + tool := outputCaptureTool("summary", task.OutputSpec{Type: "string"}) + var schema jsonschema.Schema + if err := json.Unmarshal(tool.Parameters, &schema); err != nil { + t.Fatalf("tool schema: %v", err) + } + resolved, err := schema.Resolve(nil) + if err != nil { + t.Fatalf("resolve: %v", err) + } + tests := []struct { + name string + value any + wantErr bool + }{ + {name: "a string value", value: "text"}, + {name: "null means not written", value: nil}, + {name: "a number is not a string", value: 1, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := resolved.Validate(map[string]any{"value": tt.value}) + if (err != nil) != tt.wantErr { + t.Fatalf("Validate(%#v) err = %v, want error %v", tt.value, err, tt.wantErr) + } + }) + } +}