fix(agentexec): enforce capture tool result schemas
This commit is contained in:
1 parent
ef8d9af469
commit
0a47a95cc3
7 files changed
+279
-2
No files matched your search
@@ -34,7 +34,6 @@ require (
|
|||||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||||
github.com/golang/protobuf v1.5.4 // indirect
|
github.com/golang/protobuf v1.5.4 // indirect
|
||||||
github.com/google/go-tpm v0.9.8 // indirect
|
github.com/google/go-tpm v0.9.8 // indirect
|
||||||
github.com/google/jsonschema-go v0.4.3 // indirect
|
|
||||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
@@ -100,6 +99,7 @@ require (
|
|||||||
github.com/go-git/go-billy/v6 v6.0.0-alpha.2
|
github.com/go-git/go-billy/v6 v6.0.0-alpha.2
|
||||||
github.com/go-git/go-git/v6 v6.0.0-alpha.5
|
github.com/go-git/go-git/v6 v6.0.0-alpha.5
|
||||||
github.com/go-telegram/bot v1.27.0
|
github.com/go-telegram/bot v1.27.0
|
||||||
|
github.com/google/jsonschema-go v0.4.3
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/hashicorp/go-hclog v1.6.3
|
github.com/hashicorp/go-hclog v1.6.3
|
||||||
github.com/hashicorp/go-plugin v1.8.0
|
github.com/hashicorp/go-plugin v1.8.0
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/google/jsonschema-go/jsonschema"
|
||||||
|
|
||||||
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -74,9 +76,52 @@ func ValidateCaptureArgs(spec agentrun.CaptureTool, value json.RawMessage) (stri
|
|||||||
return rejection, false
|
return rejection, false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// The derived checks above cover the common cases with friendly wording;
|
||||||
|
// the schema is what enforces the rest of what the tool advertised --
|
||||||
|
// enums, numeric types, nested objects, array shapes -- so a call the
|
||||||
|
// agent could only make valid is rejected with the reason, not recorded.
|
||||||
|
return checkSchema(spec, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkSchema validates the arguments against the tool's advertised JSON
|
||||||
|
// Schema. A tool with no schema, or one that will not resolve, is accepted
|
||||||
|
// here: a schema that never resolves is a definition bug, refused where the
|
||||||
|
// tool is built (workflow's agentResultTool), so the agent is never blamed
|
||||||
|
// for it.
|
||||||
|
func checkSchema(spec agentrun.CaptureTool, value json.RawMessage) (string, bool) {
|
||||||
|
if len(spec.Parameters) == 0 {
|
||||||
|
return "", true
|
||||||
|
}
|
||||||
|
resolved, err := resolveCaptureSchema(spec.Parameters)
|
||||||
|
if err != nil {
|
||||||
|
return "", true
|
||||||
|
}
|
||||||
|
var instance any
|
||||||
|
if err := json.Unmarshal(value, &instance); err != nil {
|
||||||
|
return spec.Name + " rejected: arguments must be JSON", false
|
||||||
|
}
|
||||||
|
if err := resolved.Validate(instance); err != nil {
|
||||||
|
return fmt.Sprintf("%s rejected: %v", spec.Name, err), false
|
||||||
|
}
|
||||||
return "", true
|
return "", true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveCaptureSchema parses and resolves a capture tool's JSON Schema. The
|
||||||
|
// workflow parser resolves the same schema when it builds the tool, so an
|
||||||
|
// unusable schema is refused at definition time rather than silently skipped
|
||||||
|
// here.
|
||||||
|
func resolveCaptureSchema(params json.RawMessage) (*jsonschema.Resolved, error) {
|
||||||
|
var schema jsonschema.Schema
|
||||||
|
if err := json.Unmarshal(params, &schema); err != nil {
|
||||||
|
return nil, fmt.Errorf("capture tool schema: %w", err)
|
||||||
|
}
|
||||||
|
resolved, err := schema.Resolve(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("capture tool schema: %w", err)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
func checkRequiredFields(spec agentrun.CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
func checkRequiredFields(spec agentrun.CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
||||||
for _, field := range spec.RequiredFields {
|
for _, field := range spec.RequiredFields {
|
||||||
if _, ok := object[field]; !ok {
|
if _, ok := object[field]; !ok {
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package agentexec
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
// resultSchema is the shape of a shipped planner result (triage's), with a
|
||||||
|
// nested object and an array so every constraint kind the agent can violate
|
||||||
|
// is exercised.
|
||||||
|
const resultSchema = `{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"needs_code_change": {"type": "boolean"},
|
||||||
|
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"]},
|
||||||
|
"reasons": {"type": "string"},
|
||||||
|
"labels": {"type": "array", "items": {"type": "string"}, "maxItems": 2},
|
||||||
|
"plan": {"type": "object", "properties": {"risk": {"type": "integer"}}, "required": ["risk"]}
|
||||||
|
},
|
||||||
|
"required": ["needs_code_change", "workflow", "reasons"]
|
||||||
|
}`
|
||||||
|
|
||||||
|
// TestValidateCaptureArgsEnforcesSchema pins that the advertised schema is
|
||||||
|
// enforced, not merely offered to the agent: an enum, a numeric type, an
|
||||||
|
// array's item type and length, and a nested object's shape all reject with a
|
||||||
|
// message naming what was wrong, so the agent can correct it.
|
||||||
|
func TestValidateCaptureArgsEnforcesSchema(t *testing.T) {
|
||||||
|
base := `"needs_code_change": true, "workflow": "tdd", "reasons": "why"`
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
args string
|
||||||
|
wantOK bool
|
||||||
|
wantText string
|
||||||
|
}{
|
||||||
|
{name: "valid", args: `{` + base + `}`, wantOK: true},
|
||||||
|
{name: "enum violation", args: `{` + base + `, "workflow": "banana"}`, wantText: "workflow"},
|
||||||
|
{name: "wrong boolean type", args: `{"needs_code_change": "yes", "workflow": "tdd", "reasons": "why"}`, wantText: "needs_code_change"},
|
||||||
|
{name: "string where integer expected", args: `{` + base + `, "plan": {"risk": "high"}}`, wantText: "plan"},
|
||||||
|
{name: "array item type", args: `{` + base + `, "labels": [1, 2]}`, wantText: "labels"},
|
||||||
|
{name: "array too long", args: `{` + base + `, "labels": ["a", "b", "c"]}`, wantText: "labels"},
|
||||||
|
{name: "nested required missing", args: `{` + base + `, "plan": {}}`, wantText: "plan"},
|
||||||
|
}
|
||||||
|
spec := agentrun.CaptureTool{Name: "result", Parameters: json.RawMessage(resultSchema)}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
rejection, ok := ValidateCaptureArgs(spec, json.RawMessage(tt.args))
|
||||||
|
if ok != tt.wantOK {
|
||||||
|
t.Fatalf("ok = %v, rejection = %q; want ok %v", ok, rejection, tt.wantOK)
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(rejection, "result rejected: ") {
|
||||||
|
t.Fatalf("rejection %q does not name the tool", rejection)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rejection, tt.wantText) {
|
||||||
|
t.Fatalf("rejection %q does not mention %q", rejection, tt.wantText)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestValidateCaptureArgsWithoutSchema pins that a tool with no schema, or one
|
||||||
|
// whose schema cannot resolve, still accepts a call: the derived constraint
|
||||||
|
// lists remain the whole check, so the agent is never blocked by a tool
|
||||||
|
// definition the schema check cannot read.
|
||||||
|
func TestValidateCaptureArgsWithoutSchema(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
spec agentrun.CaptureTool
|
||||||
|
args string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{name: "no schema", spec: agentrun.CaptureTool{Name: "note"}, args: `{"anything": 1}`, wantOK: true},
|
||||||
|
{
|
||||||
|
name: "unresolvable schema is not the agent's problem",
|
||||||
|
spec: agentrun.CaptureTool{Name: "note", Parameters: json.RawMessage(`{"$ref": "#/definitions/missing"}`)},
|
||||||
|
args: `{"anything": 1}`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived checks still apply without a schema",
|
||||||
|
spec: agentrun.CaptureTool{Name: "note", NonEmptyStrings: []string{"body"}},
|
||||||
|
args: `{"body": " "}`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
_, ok := ValidateCaptureArgs(tt.spec, json.RawMessage(tt.args))
|
||||||
|
if ok != tt.wantOK {
|
||||||
|
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/google/jsonschema-go/jsonschema"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
|
|
||||||
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
"github.com/samcharles93/archie-core/internal/domain/agentrun"
|
||||||
@@ -143,6 +144,16 @@ func agentResultTool(schema map[string]any) (*agentrun.CaptureTool, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// A schema that will not resolve cannot be enforced at call time, so the
|
||||||
|
// definition is refused here rather than accepting results it never
|
||||||
|
// checked.
|
||||||
|
var resolved jsonschema.Schema
|
||||||
|
if err := json.Unmarshal(params, &resolved); err != nil {
|
||||||
|
return nil, fmt.Errorf("must be a valid JSON Schema: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := resolved.Resolve(nil); err != nil {
|
||||||
|
return nil, fmt.Errorf("must be a valid JSON Schema: %w", err)
|
||||||
|
}
|
||||||
var required []string
|
var required []string
|
||||||
if list, ok := schema["required"].([]any); ok {
|
if list, ok := schema["required"].([]any); ok {
|
||||||
for _, field := range list {
|
for _, field := range list {
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
package workflow
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestAgentResultToolSchema covers how an agent.run step's result schema
|
||||||
|
// becomes a capture tool: none stays none, a good schema keeps its derived
|
||||||
|
// constraints, and a schema that would not resolve is refused at definition
|
||||||
|
// time rather than producing a tool the run can never satisfy.
|
||||||
|
func TestAgentResultToolSchema(t *testing.T) {
|
||||||
|
valid := map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"needs_change": map[string]any{"type": "boolean"},
|
||||||
|
"workflow": map[string]any{"type": "string", "enum": []any{"implement", "tdd"}},
|
||||||
|
"reasons": map[string]any{"type": "string"},
|
||||||
|
},
|
||||||
|
"required": []any{"needs_change", "workflow", "reasons"},
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
schema map[string]any
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{name: "no schema", schema: nil},
|
||||||
|
{name: "valid schema", schema: valid},
|
||||||
|
{name: "not an object", schema: map[string]any{"type": "string"}, wantErr: "must be a JSON Schema with type: object"},
|
||||||
|
{
|
||||||
|
name: "unresolvable reference",
|
||||||
|
schema: map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{"x": map[string]any{"$ref": "#/definitions/missing"}},
|
||||||
|
},
|
||||||
|
wantErr: "must be a valid JSON Schema",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
tool, err := agentResultTool(tt.schema)
|
||||||
|
if tt.wantErr != "" {
|
||||||
|
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||||
|
t.Fatalf("err = %v, want containing %q", err, tt.wantErr)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if tt.schema == nil {
|
||||||
|
if tool != nil {
|
||||||
|
t.Fatalf("tool = %#v, want nil for no schema", tool)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if tool == nil {
|
||||||
|
t.Fatal("tool = nil, want a capture tool")
|
||||||
|
}
|
||||||
|
if len(tool.RequiredFields) != 3 {
|
||||||
|
t.Fatalf("RequiredFields = %v, want the schema's three required fields", tool.RequiredFields)
|
||||||
|
}
|
||||||
|
if !containsString(tool.NonEmptyStrings, "reasons") && !containsString(tool.NonEmptyStrings, "workflow") {
|
||||||
|
t.Fatalf("NonEmptyStrings = %v, want the required string fields", tool.NonEmptyStrings)
|
||||||
|
}
|
||||||
|
if !containsString(tool.BooleanFields, "needs_change") {
|
||||||
|
t.Fatalf("BooleanFields = %v, want needs_change", tool.BooleanFields)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsString(list []string, want string) bool {
|
||||||
|
return slices.Contains(list, want)
|
||||||
|
}
|
||||||
@@ -67,7 +67,11 @@ func (tc *TaskContext) WriteOutput(name string, value any) error {
|
|||||||
func outputCaptureTool(name string, spec task.OutputSpec) agentrun.CaptureTool {
|
func outputCaptureTool(name string, spec task.OutputSpec) agentrun.CaptureTool {
|
||||||
value := map[string]any{"description": fmt.Sprintf("The value of the %q output.", name)}
|
value := map[string]any{"description": fmt.Sprintf("The value of the %q output.", name)}
|
||||||
if typ := jsonSchemaType(spec.Type); typ != "" {
|
if typ := jsonSchemaType(spec.Type); typ != "" {
|
||||||
value["type"] = typ
|
// Null is how the agent withdraws a value it cannot supply, and the
|
||||||
|
// run reads a null value as "not written" (decodeCapturedOutput), so
|
||||||
|
// the advertised schema has to permit it or the call would be
|
||||||
|
// rejected for the value the contract assigns a meaning to.
|
||||||
|
value["type"] = []string{typ, "null"}
|
||||||
}
|
}
|
||||||
params, err := json.Marshal(map[string]any{
|
params, err := json.Marshal(map[string]any{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package workflow
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/google/jsonschema-go/jsonschema"
|
||||||
|
|
||||||
|
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestOutputCaptureToolSchemaAllowsNull pins the advertised schema of a
|
||||||
|
// declared output against the contract the run applies: a typed value, or a
|
||||||
|
// null that means "not written" (decodeCapturedOutput). Enforcing the schema
|
||||||
|
// must not reject the null the contract gives a meaning to.
|
||||||
|
func TestOutputCaptureToolSchemaAllowsNull(t *testing.T) {
|
||||||
|
tool := outputCaptureTool("summary", task.OutputSpec{Type: "string"})
|
||||||
|
var schema jsonschema.Schema
|
||||||
|
if err := json.Unmarshal(tool.Parameters, &schema); err != nil {
|
||||||
|
t.Fatalf("tool schema: %v", err)
|
||||||
|
}
|
||||||
|
resolved, err := schema.Resolve(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve: %v", err)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "a string value", value: "text"},
|
||||||
|
{name: "null means not written", value: nil},
|
||||||
|
{name: "a number is not a string", value: 1, wantErr: true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := resolved.Validate(map[string]any{"value": tt.value})
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Fatalf("Validate(%#v) err = %v, want error %v", tt.value, err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user