fix(agentexec): enforce capture tool result schemas

This commit is contained in:
sam committed 2026-10-08 22:30:12 +11:00
1 parent ef8d9af469
commit 0a47a95cc3
7 files changed
+279 -2

No files matched your search

+1 -1
View File
@@ -34,7 +34,6 @@ require (
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/google/jsonschema-go v0.4.3 // indirect
github.com/hashicorp/yamux v0.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -100,6 +99,7 @@ require (
github.com/go-git/go-billy/v6 v6.0.0-alpha.2
github.com/go-git/go-git/v6 v6.0.0-alpha.5
github.com/go-telegram/bot v1.27.0
github.com/google/jsonschema-go v0.4.3
github.com/google/uuid v1.6.0
github.com/hashicorp/go-hclog v1.6.3
github.com/hashicorp/go-plugin v1.8.0
+45
View File
@@ -8,6 +8,8 @@ import (
"slices"
"strings"
"github.com/google/jsonschema-go/jsonschema"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
)
@@ -74,9 +76,52 @@ func ValidateCaptureArgs(spec agentrun.CaptureTool, value json.RawMessage) (stri
return rejection, false
}
}
// The derived checks above cover the common cases with friendly wording;
// the schema is what enforces the rest of what the tool advertised --
// enums, numeric types, nested objects, array shapes -- so a call the
// agent could only make valid is rejected with the reason, not recorded.
return checkSchema(spec, value)
}
// checkSchema validates the arguments against the tool's advertised JSON
// Schema. A tool with no schema, or one that will not resolve, is accepted
// here: a schema that never resolves is a definition bug, refused where the
// tool is built (workflow's agentResultTool), so the agent is never blamed
// for it.
func checkSchema(spec agentrun.CaptureTool, value json.RawMessage) (string, bool) {
if len(spec.Parameters) == 0 {
return "", true
}
resolved, err := resolveCaptureSchema(spec.Parameters)
if err != nil {
return "", true
}
var instance any
if err := json.Unmarshal(value, &instance); err != nil {
return spec.Name + " rejected: arguments must be JSON", false
}
if err := resolved.Validate(instance); err != nil {
return fmt.Sprintf("%s rejected: %v", spec.Name, err), false
}
return "", true
}
// resolveCaptureSchema parses and resolves a capture tool's JSON Schema. The
// workflow parser resolves the same schema when it builds the tool, so an
// unusable schema is refused at definition time rather than silently skipped
// here.
func resolveCaptureSchema(params json.RawMessage) (*jsonschema.Resolved, error) {
var schema jsonschema.Schema
if err := json.Unmarshal(params, &schema); err != nil {
return nil, fmt.Errorf("capture tool schema: %w", err)
}
resolved, err := schema.Resolve(nil)
if err != nil {
return nil, fmt.Errorf("capture tool schema: %w", err)
}
return resolved, nil
}
func checkRequiredFields(spec agentrun.CaptureTool, object map[string]json.RawMessage) (string, bool) {
for _, field := range spec.RequiredFields {
if _, ok := object[field]; !ok {
+98
View File
@@ -0,0 +1,98 @@
package agentexec
import (
"encoding/json"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
)
// resultSchema is the shape of a shipped planner result (triage's), with a
// nested object and an array so every constraint kind the agent can violate
// is exercised.
const resultSchema = `{
"type": "object",
"properties": {
"needs_code_change": {"type": "boolean"},
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"]},
"reasons": {"type": "string"},
"labels": {"type": "array", "items": {"type": "string"}, "maxItems": 2},
"plan": {"type": "object", "properties": {"risk": {"type": "integer"}}, "required": ["risk"]}
},
"required": ["needs_code_change", "workflow", "reasons"]
}`
// TestValidateCaptureArgsEnforcesSchema pins that the advertised schema is
// enforced, not merely offered to the agent: an enum, a numeric type, an
// array's item type and length, and a nested object's shape all reject with a
// message naming what was wrong, so the agent can correct it.
func TestValidateCaptureArgsEnforcesSchema(t *testing.T) {
base := `"needs_code_change": true, "workflow": "tdd", "reasons": "why"`
tests := []struct {
name string
args string
wantOK bool
wantText string
}{
{name: "valid", args: `{` + base + `}`, wantOK: true},
{name: "enum violation", args: `{` + base + `, "workflow": "banana"}`, wantText: "workflow"},
{name: "wrong boolean type", args: `{"needs_code_change": "yes", "workflow": "tdd", "reasons": "why"}`, wantText: "needs_code_change"},
{name: "string where integer expected", args: `{` + base + `, "plan": {"risk": "high"}}`, wantText: "plan"},
{name: "array item type", args: `{` + base + `, "labels": [1, 2]}`, wantText: "labels"},
{name: "array too long", args: `{` + base + `, "labels": ["a", "b", "c"]}`, wantText: "labels"},
{name: "nested required missing", args: `{` + base + `, "plan": {}}`, wantText: "plan"},
}
spec := agentrun.CaptureTool{Name: "result", Parameters: json.RawMessage(resultSchema)}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rejection, ok := ValidateCaptureArgs(spec, json.RawMessage(tt.args))
if ok != tt.wantOK {
t.Fatalf("ok = %v, rejection = %q; want ok %v", ok, rejection, tt.wantOK)
}
if ok {
return
}
if !strings.HasPrefix(rejection, "result rejected: ") {
t.Fatalf("rejection %q does not name the tool", rejection)
}
if !strings.Contains(rejection, tt.wantText) {
t.Fatalf("rejection %q does not mention %q", rejection, tt.wantText)
}
})
}
}
// TestValidateCaptureArgsWithoutSchema pins that a tool with no schema, or one
// whose schema cannot resolve, still accepts a call: the derived constraint
// lists remain the whole check, so the agent is never blocked by a tool
// definition the schema check cannot read.
func TestValidateCaptureArgsWithoutSchema(t *testing.T) {
tests := []struct {
name string
spec agentrun.CaptureTool
args string
wantOK bool
}{
{name: "no schema", spec: agentrun.CaptureTool{Name: "note"}, args: `{"anything": 1}`, wantOK: true},
{
name: "unresolvable schema is not the agent's problem",
spec: agentrun.CaptureTool{Name: "note", Parameters: json.RawMessage(`{"$ref": "#/definitions/missing"}`)},
args: `{"anything": 1}`,
wantOK: true,
},
{
name: "derived checks still apply without a schema",
spec: agentrun.CaptureTool{Name: "note", NonEmptyStrings: []string{"body"}},
args: `{"body": " "}`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, ok := ValidateCaptureArgs(tt.spec, json.RawMessage(tt.args))
if ok != tt.wantOK {
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
}
})
}
}
+11
View File
@@ -8,6 +8,7 @@ import (
"slices"
"strings"
"github.com/google/jsonschema-go/jsonschema"
"gopkg.in/yaml.v3"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
@@ -143,6 +144,16 @@ func agentResultTool(schema map[string]any) (*agentrun.CaptureTool, error) {
if err != nil {
return nil, err
}
// A schema that will not resolve cannot be enforced at call time, so the
// definition is refused here rather than accepting results it never
// checked.
var resolved jsonschema.Schema
if err := json.Unmarshal(params, &resolved); err != nil {
return nil, fmt.Errorf("must be a valid JSON Schema: %w", err)
}
if _, err := resolved.Resolve(nil); err != nil {
return nil, fmt.Errorf("must be a valid JSON Schema: %w", err)
}
var required []string
if list, ok := schema["required"].([]any); ok {
for _, field := range list {
@@ -0,0 +1,76 @@
package workflow
import (
"slices"
"strings"
"testing"
)
// TestAgentResultToolSchema covers how an agent.run step's result schema
// becomes a capture tool: none stays none, a good schema keeps its derived
// constraints, and a schema that would not resolve is refused at definition
// time rather than producing a tool the run can never satisfy.
func TestAgentResultToolSchema(t *testing.T) {
valid := map[string]any{
"type": "object",
"properties": map[string]any{
"needs_change": map[string]any{"type": "boolean"},
"workflow": map[string]any{"type": "string", "enum": []any{"implement", "tdd"}},
"reasons": map[string]any{"type": "string"},
},
"required": []any{"needs_change", "workflow", "reasons"},
}
tests := []struct {
name string
schema map[string]any
wantErr string
}{
{name: "no schema", schema: nil},
{name: "valid schema", schema: valid},
{name: "not an object", schema: map[string]any{"type": "string"}, wantErr: "must be a JSON Schema with type: object"},
{
name: "unresolvable reference",
schema: map[string]any{
"type": "object",
"properties": map[string]any{"x": map[string]any{"$ref": "#/definitions/missing"}},
},
wantErr: "must be a valid JSON Schema",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tool, err := agentResultTool(tt.schema)
if tt.wantErr != "" {
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if tt.schema == nil {
if tool != nil {
t.Fatalf("tool = %#v, want nil for no schema", tool)
}
return
}
if tool == nil {
t.Fatal("tool = nil, want a capture tool")
}
if len(tool.RequiredFields) != 3 {
t.Fatalf("RequiredFields = %v, want the schema's three required fields", tool.RequiredFields)
}
if !containsString(tool.NonEmptyStrings, "reasons") && !containsString(tool.NonEmptyStrings, "workflow") {
t.Fatalf("NonEmptyStrings = %v, want the required string fields", tool.NonEmptyStrings)
}
if !containsString(tool.BooleanFields, "needs_change") {
t.Fatalf("BooleanFields = %v, want needs_change", tool.BooleanFields)
}
})
}
}
func containsString(list []string, want string) bool {
return slices.Contains(list, want)
}
+5 -1
View File
@@ -67,7 +67,11 @@ func (tc *TaskContext) WriteOutput(name string, value any) error {
func outputCaptureTool(name string, spec task.OutputSpec) agentrun.CaptureTool {
value := map[string]any{"description": fmt.Sprintf("The value of the %q output.", name)}
if typ := jsonSchemaType(spec.Type); typ != "" {
value["type"] = typ
// Null is how the agent withdraws a value it cannot supply, and the
// run reads a null value as "not written" (decodeCapturedOutput), so
// the advertised schema has to permit it or the call would be
// rejected for the value the contract assigns a meaning to.
value["type"] = []string{typ, "null"}
}
params, err := json.Marshal(map[string]any{
"type": "object",
+43
View File
@@ -0,0 +1,43 @@
package workflow
import (
"encoding/json"
"testing"
"github.com/google/jsonschema-go/jsonschema"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
)
// TestOutputCaptureToolSchemaAllowsNull pins the advertised schema of a
// declared output against the contract the run applies: a typed value, or a
// null that means "not written" (decodeCapturedOutput). Enforcing the schema
// must not reject the null the contract gives a meaning to.
func TestOutputCaptureToolSchemaAllowsNull(t *testing.T) {
tool := outputCaptureTool("summary", task.OutputSpec{Type: "string"})
var schema jsonschema.Schema
if err := json.Unmarshal(tool.Parameters, &schema); err != nil {
t.Fatalf("tool schema: %v", err)
}
resolved, err := schema.Resolve(nil)
if err != nil {
t.Fatalf("resolve: %v", err)
}
tests := []struct {
name string
value any
wantErr bool
}{
{name: "a string value", value: "text"},
{name: "null means not written", value: nil},
{name: "a number is not a string", value: 1, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := resolved.Validate(map[string]any{"value": tt.value})
if (err != nil) != tt.wantErr {
t.Fatalf("Validate(%#v) err = %v, want error %v", tt.value, err, tt.wantErr)
}
})
}
}