From fe7ee44dcc5e7164c6a19ee101bc5dbe848ed250 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 19 Sep 2026 00:33:39 +1000 Subject: [PATCH] fix(workflow): require a workflow from triage when a code change is needed The selection criteria alone left workflow optional and defaulted an omitted or unrecognised value to implement, so a decide call could still skip the decision and send an unsettled capability straight to the builder. Omission is now a retryable rejection: CaptureTool gains RequiredWhenTrue, a conditional requirement a flat RequiredFields entry cannot express, because the close-without-change branch has no workflow to name and must not be made to invent one. A rejected call is not recorded, so it does not consume MaxCalls and the model retries with the field. A value outside the enum now fails the stage rather than degrading to implement: reaching OnResult with one means the model ignored its own schema. Splits validateCaptureArgs into per-constraint checks (cyclop) and carries the regenerated contracts.json. --- docs/data/generated/contracts.json | 10 +++ internal/agentexec/capture_validation_test.go | 62 +++++++++++++++++ internal/agentexec/loop_runner.go | 67 ++++++++++++++++--- internal/agentexec/protocol.go | 10 ++- internal/domain/workflow/triage.go | 16 +++-- internal/domain/workflow/triage_test.go | 43 +++++++++--- 6 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 internal/agentexec/capture_validation_test.go diff --git a/docs/data/generated/contracts.json b/docs/data/generated/contracts.json index 6101879dd..159b07e54 100644 --- a/docs/data/generated/contracts.json +++ b/docs/data/generated/contracts.json @@ -191,6 +191,16 @@ "type": "string" }, "type": "array" + }, + "required_when_true": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array" + }, + "description": "RequiredWhenTrue makes fields required only when a boolean field in\nthe same call is true, keyed by that boolean's name. It exists because\na flat RequiredFields entry cannot express \"name a workflow, but only\nwhen you have said one is needed\": the alternatives were forcing a\nmeaningless answer on the other branch, or accepting the omission and\ndefaulting it, which lets the model skip the decision entirely.\nA named field must also be a non-empty string when the trigger is true.", + "type": "object" } }, "required": [ diff --git a/internal/agentexec/capture_validation_test.go b/internal/agentexec/capture_validation_test.go new file mode 100644 index 000000000..ff7dfef0b --- /dev/null +++ b/internal/agentexec/capture_validation_test.go @@ -0,0 +1,62 @@ +package agentexec + +import ( + "encoding/json" + "strings" + "testing" +) + +// A capture tool sometimes needs a field only under a condition the same call +// carries: triage must name a workflow when it says a code change is needed, +// and must not be made to invent one when it says the opposite. A flat +// RequiredFields entry cannot say that, so the choice used to be between a +// meaningless forced answer and a silent default. +func TestValidateCaptureArgsConditionalRequirement(t *testing.T) { + spec := CaptureTool{ + Name: "decide", + RequiredFields: []string{"needs_code_change"}, + RequiredWhenTrue: map[string][]string{ + "needs_code_change": {"workflow"}, + }, + } + + tests := []struct { + name string + args string + wantOK bool + wantRejected string + }{ + { + name: "trigger true and field present", + args: `{"needs_code_change": true, "workflow": "feasibility"}`, + wantOK: true, + }, + { + name: "trigger true and field missing", + args: `{"needs_code_change": true}`, + wantRejected: "workflow", + }, + { + name: "trigger true and field empty", + args: `{"needs_code_change": true, "workflow": " "}`, + wantRejected: "workflow", + }, + { + name: "trigger false, field not required", + args: `{"needs_code_change": false}`, + wantOK: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rejection, ok := validateCaptureArgs(spec, json.RawMessage(tt.args)) + if ok != tt.wantOK { + t.Fatalf("validateCaptureArgs ok = %v, want %v (rejection %q)", ok, tt.wantOK, rejection) + } + if tt.wantRejected != "" && !strings.Contains(rejection, tt.wantRejected) { + t.Fatalf("rejection %q does not name %q", rejection, tt.wantRejected) + } + }) + } +} diff --git a/internal/agentexec/loop_runner.go b/internal/agentexec/loop_runner.go index 6695310b2..ad9e87776 100644 --- a/internal/agentexec/loop_runner.go +++ b/internal/agentexec/loop_runner.go @@ -7,6 +7,7 @@ import ( "log/slog" "maps" "path/filepath" + "slices" "strings" "github.com/samcharles93/ai-sdk/agentloop" @@ -283,32 +284,78 @@ func makeCaptureHandler(spec CaptureTool, captures map[string][]json.RawMessage) // validateCaptureArgs checks that the capture tool arguments satisfy all // constraints. Returns a rejection message and false on failure, or "" and -// true on success. +// true on success. Each constraint is its own check so adding one does not +// grow a single branching function. func validateCaptureArgs(spec CaptureTool, value json.RawMessage) (string, bool) { var object map[string]json.RawMessage if err := json.Unmarshal(value, &object); err != nil { return spec.Name + " rejected: arguments must be a JSON object", false //nolint:nilerr // the agent loop must see malformed tool arguments as feedback it can correct, not as a failed tool call } + for _, check := range []func(CaptureTool, map[string]json.RawMessage) (string, bool){ + checkRequiredFields, + checkNonEmptyStrings, + checkBooleanFields, + checkRequiredWhenTrue, + } { + if rejection, ok := check(spec, object); !ok { + return rejection, false + } + } + return "", true +} + +func checkRequiredFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) { for _, field := range spec.RequiredFields { if _, ok := object[field]; !ok { return fmt.Sprintf("%s rejected: %s is required", spec.Name, field), false } } + return "", true +} + +func checkNonEmptyStrings(spec CaptureTool, object map[string]json.RawMessage) (string, bool) { for _, field := range spec.NonEmptyStrings { - var text string - if raw, ok := object[field]; !ok || json.Unmarshal(raw, &text) != nil || strings.TrimSpace(text) == "" { - return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument - } - } - for _, field := range spec.BooleanFields { - var val bool - if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil { - return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument + if !isNonEmptyString(object, field) { + return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false } } return "", true } +func checkBooleanFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) { + for _, field := range spec.BooleanFields { + var val bool + if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil { + return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false + } + } + return "", true +} + +// checkRequiredWhenTrue enforces the conditional requirements. Triggers are +// visited in sorted order so a call violating two of them names the same one +// every run, rather than a different one on each retry. +func checkRequiredWhenTrue(spec CaptureTool, object map[string]json.RawMessage) (string, bool) { + for _, trigger := range slices.Sorted(maps.Keys(spec.RequiredWhenTrue)) { + var on bool + if raw, ok := object[trigger]; !ok || json.Unmarshal(raw, &on) != nil || !on { + continue + } + for _, field := range spec.RequiredWhenTrue[trigger] { + if !isNonEmptyString(object, field) { + return fmt.Sprintf("%s rejected: %s is required when %s is true", spec.Name, field, trigger), false + } + } + } + return "", true +} + +func isNonEmptyString(object map[string]json.RawMessage, field string) bool { + var text string + raw, ok := object[field] + return ok && json.Unmarshal(raw, &text) == nil && strings.TrimSpace(text) != "" +} + // scriptToolSet exposes run_go_script: interpreting a Yaegi Go script // bundled with a skill (.agents/skills//scripts/*.go) or living // anywhere else in the workspace, and returning what it printed. This is diff --git a/internal/agentexec/protocol.go b/internal/agentexec/protocol.go index 090e6cbd3..21d6fce98 100644 --- a/internal/agentexec/protocol.go +++ b/internal/agentexec/protocol.go @@ -52,7 +52,15 @@ type CaptureTool struct { RequiredFields []string `json:"required_fields,omitempty"` NonEmptyStrings []string `json:"non_empty_strings,omitempty"` BooleanFields []string `json:"boolean_fields,omitempty"` - MaxCalls int `json:"max_calls,omitempty"` + // RequiredWhenTrue makes fields required only when a boolean field in + // the same call is true, keyed by that boolean's name. It exists because + // a flat RequiredFields entry cannot express "name a workflow, but only + // when you have said one is needed": the alternatives were forcing a + // meaningless answer on the other branch, or accepting the omission and + // defaulting it, which lets the model skip the decision entirely. + // A named field must also be a non-empty string when the trigger is true. + RequiredWhenTrue map[string][]string `json:"required_when_true,omitempty"` + MaxCalls int `json:"max_calls,omitempty"` } // Request is the complete, serializable input for one agent stage. The diff --git a/internal/domain/workflow/triage.go b/internal/domain/workflow/triage.go index 9bddc9e79..7830ef76b 100644 --- a/internal/domain/workflow/triage.go +++ b/internal/domain/workflow/triage.go @@ -9,8 +9,7 @@ import ( ) // triageWorkflowNames is the set of workflows triage may hand a task to. -// Unrecognized or missing classifier output falls back to "implement" -- -// see triageDecideCaptureTools' schema description and Triage's OnResult. +// A name outside it is refused, not defaulted -- see Triage's OnResult. var triageWorkflowNames = map[string]bool{ "implement": true, "tdd": true, @@ -74,9 +73,14 @@ func Triage() Workflow { tc.Outcome = Outcome{Status: StatusMerged, Detail: "triaged: no code change needed -- " + captured.Reasons} return nil } + // No fallback. The decide tool already refuses a call that + // needs a code change and names no workflow, so reaching + // here with an unrecognised one means the model ignored + // its own enum; defaulting that to implement is how an + // unsettled capability used to reach the builder. target := captured.Workflow if !triageWorkflowNames[target] { - target = "implement" + return fmt.Errorf("triage chose workflow %q, which is not one of implement, tdd or feasibility", target) } tc.Task.Workflow = target tc.Outcome = Outcome{ @@ -105,7 +109,11 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool { return []agentexec.CaptureTool{{ Name: "decide", Description: "Record the triage verdict. Call exactly once, before finish.", Parameters: params, RequiredFields: []string{"needs_code_change", "reasons"}, - NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, MaxCalls: 1, + NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, + // Conditional, not flat: a task needing no code change has no + // workflow to name, and forcing one would be a meaningless answer. + RequiredWhenTrue: map[string][]string{"needs_code_change": {"workflow"}}, + MaxCalls: 1, }} } diff --git a/internal/domain/workflow/triage_test.go b/internal/domain/workflow/triage_test.go index 1e79d6f3f..b84622a12 100644 --- a/internal/domain/workflow/triage_test.go +++ b/internal/domain/workflow/triage_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "log/slog" + "slices" "strings" "testing" @@ -141,10 +142,13 @@ func TestTriageRequeuesUnderChosenWorkflow(t *testing.T) { } } -// TestTriageDefaultsToImplementForUnrecognizedWorkflow confirms a missing -// or unrecognized workflow choice degrades to "implement" rather than -// producing an unroutable task. -func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) { +// Defaulting an unrecognised workflow to "implement" let the classifier skip +// the decision and send an unsettled capability straight to coding, which is +// the failure the selection criteria exist to prevent. Omission is caught +// earlier, by the decide tool's conditional requirement, so the model is +// told and retries; a value that is not in the enum at all means the model +// ignored its schema, and parking is the honest outcome. +func TestTriageRefusesAnUnrecognizedWorkflow(t *testing.T) { runner := agentRunnerFunc(func(_ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) { return agentexec.Result{ Version: agentexec.ProtocolVersion, @@ -152,7 +156,7 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) { Attempt: req.Attempt, Stage: req.Stage, Status: agentexec.StatusPassed, - Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure, defaulting")}, + Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure")}, }, nil }) tc := &TaskContext{ @@ -163,11 +167,32 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) { Log: slog.New(slog.DiscardHandler), } - if err := classifyStage().Run(context.Background(), tc); err != nil { - t.Fatalf("classify stage = %v, want nil", err) + err := classifyStage().Run(context.Background(), tc) + if err == nil { + t.Fatal("classify stage accepted an unrecognised workflow") } - if tc.Task.Workflow != "implement" { - t.Fatalf("Task.Workflow = %q, want %q", tc.Task.Workflow, "implement") + if !strings.Contains(err.Error(), "not-a-real-workflow") { + t.Fatalf("error %q does not name the rejected workflow", err) + } + if tc.Task.Workflow != "" { + t.Fatalf("Task.Workflow = %q, want it left unset", tc.Task.Workflow) + } +} + +// The decide tool must demand a workflow exactly when one is needed, so an +// omission is a retryable rejection rather than a silent default, and the +// close-without-change branch is not made to invent an answer. +func TestTriageRequiresAWorkflowOnlyWhenACodeChangeIsNeeded(t *testing.T) { + tools := triageDecideCaptureTools(nil) + if len(tools) != 1 { + t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools)) + } + required := tools[0].RequiredWhenTrue["needs_code_change"] + if !slices.Contains(required, "workflow") { + t.Fatalf("decide requires %v when needs_code_change is true, want it to include \"workflow\"", required) + } + if slices.Contains(tools[0].RequiredFields, "workflow") { + t.Error("decide requires workflow unconditionally; the close-without-change branch has no workflow to give") } }