fix(workflow): require a workflow from triage when a code change is needed
The selection criteria alone left workflow optional and defaulted an omitted or unrecognised value to implement, so a decide call could still skip the decision and send an unsettled capability straight to the builder. Omission is now a retryable rejection: CaptureTool gains RequiredWhenTrue, a conditional requirement a flat RequiredFields entry cannot express, because the close-without-change branch has no workflow to name and must not be made to invent one. A rejected call is not recorded, so it does not consume MaxCalls and the model retries with the field. A value outside the enum now fails the stage rather than degrading to implement: reaching OnResult with one means the model ignored its own schema. Splits validateCaptureArgs into per-constraint checks (cyclop) and carries the regenerated contracts.json.
This commit is contained in:
6 files changed
+184
-24
No files matched your search
@@ -191,6 +191,16 @@
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"required_when_true": {
|
||||
"additionalProperties": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"description": "RequiredWhenTrue makes fields required only when a boolean field in\nthe same call is true, keyed by that boolean's name. It exists because\na flat RequiredFields entry cannot express \"name a workflow, but only\nwhen you have said one is needed\": the alternatives were forcing a\nmeaningless answer on the other branch, or accepting the omission and\ndefaulting it, which lets the model skip the decision entirely.\nA named field must also be a non-empty string when the trigger is true.",
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package agentexec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A capture tool sometimes needs a field only under a condition the same call
|
||||
// carries: triage must name a workflow when it says a code change is needed,
|
||||
// and must not be made to invent one when it says the opposite. A flat
|
||||
// RequiredFields entry cannot say that, so the choice used to be between a
|
||||
// meaningless forced answer and a silent default.
|
||||
func TestValidateCaptureArgsConditionalRequirement(t *testing.T) {
|
||||
spec := CaptureTool{
|
||||
Name: "decide",
|
||||
RequiredFields: []string{"needs_code_change"},
|
||||
RequiredWhenTrue: map[string][]string{
|
||||
"needs_code_change": {"workflow"},
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
args string
|
||||
wantOK bool
|
||||
wantRejected string
|
||||
}{
|
||||
{
|
||||
name: "trigger true and field present",
|
||||
args: `{"needs_code_change": true, "workflow": "feasibility"}`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "trigger true and field missing",
|
||||
args: `{"needs_code_change": true}`,
|
||||
wantRejected: "workflow",
|
||||
},
|
||||
{
|
||||
name: "trigger true and field empty",
|
||||
args: `{"needs_code_change": true, "workflow": " "}`,
|
||||
wantRejected: "workflow",
|
||||
},
|
||||
{
|
||||
name: "trigger false, field not required",
|
||||
args: `{"needs_code_change": false}`,
|
||||
wantOK: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rejection, ok := validateCaptureArgs(spec, json.RawMessage(tt.args))
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("validateCaptureArgs ok = %v, want %v (rejection %q)", ok, tt.wantOK, rejection)
|
||||
}
|
||||
if tt.wantRejected != "" && !strings.Contains(rejection, tt.wantRejected) {
|
||||
t.Fatalf("rejection %q does not name %q", rejection, tt.wantRejected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"maps"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/samcharles93/ai-sdk/agentloop"
|
||||
@@ -283,32 +284,78 @@ func makeCaptureHandler(spec CaptureTool, captures map[string][]json.RawMessage)
|
||||
|
||||
// validateCaptureArgs checks that the capture tool arguments satisfy all
|
||||
// constraints. Returns a rejection message and false on failure, or "" and
|
||||
// true on success.
|
||||
// true on success. Each constraint is its own check so adding one does not
|
||||
// grow a single branching function.
|
||||
func validateCaptureArgs(spec CaptureTool, value json.RawMessage) (string, bool) {
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal(value, &object); err != nil {
|
||||
return spec.Name + " rejected: arguments must be a JSON object", false //nolint:nilerr // the agent loop must see malformed tool arguments as feedback it can correct, not as a failed tool call
|
||||
}
|
||||
for _, check := range []func(CaptureTool, map[string]json.RawMessage) (string, bool){
|
||||
checkRequiredFields,
|
||||
checkNonEmptyStrings,
|
||||
checkBooleanFields,
|
||||
checkRequiredWhenTrue,
|
||||
} {
|
||||
if rejection, ok := check(spec, object); !ok {
|
||||
return rejection, false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkRequiredFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
||||
for _, field := range spec.RequiredFields {
|
||||
if _, ok := object[field]; !ok {
|
||||
return fmt.Sprintf("%s rejected: %s is required", spec.Name, field), false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkNonEmptyStrings(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
||||
for _, field := range spec.NonEmptyStrings {
|
||||
var text string
|
||||
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &text) != nil || strings.TrimSpace(text) == "" {
|
||||
return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument
|
||||
}
|
||||
}
|
||||
for _, field := range spec.BooleanFields {
|
||||
var val bool
|
||||
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil {
|
||||
return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument
|
||||
if !isNonEmptyString(object, field) {
|
||||
return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func checkBooleanFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
||||
for _, field := range spec.BooleanFields {
|
||||
var val bool
|
||||
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil {
|
||||
return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// checkRequiredWhenTrue enforces the conditional requirements. Triggers are
|
||||
// visited in sorted order so a call violating two of them names the same one
|
||||
// every run, rather than a different one on each retry.
|
||||
func checkRequiredWhenTrue(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
|
||||
for _, trigger := range slices.Sorted(maps.Keys(spec.RequiredWhenTrue)) {
|
||||
var on bool
|
||||
if raw, ok := object[trigger]; !ok || json.Unmarshal(raw, &on) != nil || !on {
|
||||
continue
|
||||
}
|
||||
for _, field := range spec.RequiredWhenTrue[trigger] {
|
||||
if !isNonEmptyString(object, field) {
|
||||
return fmt.Sprintf("%s rejected: %s is required when %s is true", spec.Name, field, trigger), false
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
func isNonEmptyString(object map[string]json.RawMessage, field string) bool {
|
||||
var text string
|
||||
raw, ok := object[field]
|
||||
return ok && json.Unmarshal(raw, &text) == nil && strings.TrimSpace(text) != ""
|
||||
}
|
||||
|
||||
// scriptToolSet exposes run_go_script: interpreting a Yaegi Go script
|
||||
// bundled with a skill (.agents/skills/<skill>/scripts/*.go) or living
|
||||
// anywhere else in the workspace, and returning what it printed. This is
|
||||
|
||||
@@ -52,7 +52,15 @@ type CaptureTool struct {
|
||||
RequiredFields []string `json:"required_fields,omitempty"`
|
||||
NonEmptyStrings []string `json:"non_empty_strings,omitempty"`
|
||||
BooleanFields []string `json:"boolean_fields,omitempty"`
|
||||
MaxCalls int `json:"max_calls,omitempty"`
|
||||
// RequiredWhenTrue makes fields required only when a boolean field in
|
||||
// the same call is true, keyed by that boolean's name. It exists because
|
||||
// a flat RequiredFields entry cannot express "name a workflow, but only
|
||||
// when you have said one is needed": the alternatives were forcing a
|
||||
// meaningless answer on the other branch, or accepting the omission and
|
||||
// defaulting it, which lets the model skip the decision entirely.
|
||||
// A named field must also be a non-empty string when the trigger is true.
|
||||
RequiredWhenTrue map[string][]string `json:"required_when_true,omitempty"`
|
||||
MaxCalls int `json:"max_calls,omitempty"`
|
||||
}
|
||||
|
||||
// Request is the complete, serializable input for one agent stage. The
|
||||
|
||||
@@ -9,8 +9,7 @@ import (
|
||||
)
|
||||
|
||||
// triageWorkflowNames is the set of workflows triage may hand a task to.
|
||||
// Unrecognized or missing classifier output falls back to "implement" --
|
||||
// see triageDecideCaptureTools' schema description and Triage's OnResult.
|
||||
// A name outside it is refused, not defaulted -- see Triage's OnResult.
|
||||
var triageWorkflowNames = map[string]bool{
|
||||
"implement": true,
|
||||
"tdd": true,
|
||||
@@ -74,9 +73,14 @@ func Triage() Workflow {
|
||||
tc.Outcome = Outcome{Status: StatusMerged, Detail: "triaged: no code change needed -- " + captured.Reasons}
|
||||
return nil
|
||||
}
|
||||
// No fallback. The decide tool already refuses a call that
|
||||
// needs a code change and names no workflow, so reaching
|
||||
// here with an unrecognised one means the model ignored
|
||||
// its own enum; defaulting that to implement is how an
|
||||
// unsettled capability used to reach the builder.
|
||||
target := captured.Workflow
|
||||
if !triageWorkflowNames[target] {
|
||||
target = "implement"
|
||||
return fmt.Errorf("triage chose workflow %q, which is not one of implement, tdd or feasibility", target)
|
||||
}
|
||||
tc.Task.Workflow = target
|
||||
tc.Outcome = Outcome{
|
||||
@@ -105,7 +109,11 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool {
|
||||
return []agentexec.CaptureTool{{
|
||||
Name: "decide", Description: "Record the triage verdict. Call exactly once, before finish.",
|
||||
Parameters: params, RequiredFields: []string{"needs_code_change", "reasons"},
|
||||
NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, MaxCalls: 1,
|
||||
NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"},
|
||||
// Conditional, not flat: a task needing no code change has no
|
||||
// workflow to name, and forcing one would be a meaningless answer.
|
||||
RequiredWhenTrue: map[string][]string{"needs_code_change": {"workflow"}},
|
||||
MaxCalls: 1,
|
||||
}}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -141,10 +142,13 @@ func TestTriageRequeuesUnderChosenWorkflow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTriageDefaultsToImplementForUnrecognizedWorkflow confirms a missing
|
||||
// or unrecognized workflow choice degrades to "implement" rather than
|
||||
// producing an unroutable task.
|
||||
func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
|
||||
// Defaulting an unrecognised workflow to "implement" let the classifier skip
|
||||
// the decision and send an unsettled capability straight to coding, which is
|
||||
// the failure the selection criteria exist to prevent. Omission is caught
|
||||
// earlier, by the decide tool's conditional requirement, so the model is
|
||||
// told and retries; a value that is not in the enum at all means the model
|
||||
// ignored its schema, and parking is the honest outcome.
|
||||
func TestTriageRefusesAnUnrecognizedWorkflow(t *testing.T) {
|
||||
runner := agentRunnerFunc(func(_ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
return agentexec.Result{
|
||||
Version: agentexec.ProtocolVersion,
|
||||
@@ -152,7 +156,7 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
|
||||
Attempt: req.Attempt,
|
||||
Stage: req.Stage,
|
||||
Status: agentexec.StatusPassed,
|
||||
Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure, defaulting")},
|
||||
Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure")},
|
||||
}, nil
|
||||
})
|
||||
tc := &TaskContext{
|
||||
@@ -163,11 +167,32 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
|
||||
if err := classifyStage().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("classify stage = %v, want nil", err)
|
||||
err := classifyStage().Run(context.Background(), tc)
|
||||
if err == nil {
|
||||
t.Fatal("classify stage accepted an unrecognised workflow")
|
||||
}
|
||||
if tc.Task.Workflow != "implement" {
|
||||
t.Fatalf("Task.Workflow = %q, want %q", tc.Task.Workflow, "implement")
|
||||
if !strings.Contains(err.Error(), "not-a-real-workflow") {
|
||||
t.Fatalf("error %q does not name the rejected workflow", err)
|
||||
}
|
||||
if tc.Task.Workflow != "" {
|
||||
t.Fatalf("Task.Workflow = %q, want it left unset", tc.Task.Workflow)
|
||||
}
|
||||
}
|
||||
|
||||
// The decide tool must demand a workflow exactly when one is needed, so an
|
||||
// omission is a retryable rejection rather than a silent default, and the
|
||||
// close-without-change branch is not made to invent an answer.
|
||||
func TestTriageRequiresAWorkflowOnlyWhenACodeChangeIsNeeded(t *testing.T) {
|
||||
tools := triageDecideCaptureTools(nil)
|
||||
if len(tools) != 1 {
|
||||
t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools))
|
||||
}
|
||||
required := tools[0].RequiredWhenTrue["needs_code_change"]
|
||||
if !slices.Contains(required, "workflow") {
|
||||
t.Fatalf("decide requires %v when needs_code_change is true, want it to include \"workflow\"", required)
|
||||
}
|
||||
if slices.Contains(tools[0].RequiredFields, "workflow") {
|
||||
t.Error("decide requires workflow unconditionally; the close-without-change branch has no workflow to give")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user