Merge pull request #870 from samcharles93/fix/label-routing-and-triage-criteria

fix: restore kind routing for namespaced labels, and give triage real criteria
This commit is contained in:
sam authored and GitHub committed 2026-09-19 00:35:05 +10:00
commit 121fbf730f
9 files changed
+315 -38

No files matched your search

+10
View File
@@ -191,6 +191,16 @@
"type": "string"
},
"type": "array"
},
"required_when_true": {
"additionalProperties": {
"items": {
"type": "string"
},
"type": "array"
},
"description": "RequiredWhenTrue makes fields required only when a boolean field in\nthe same call is true, keyed by that boolean's name. It exists because\na flat RequiredFields entry cannot express \"name a workflow, but only\nwhen you have said one is needed\": the alternatives were forcing a\nmeaningless answer on the other branch, or accepting the omission and\ndefaulting it, which lets the model skip the decision entirely.\nA named field must also be a non-empty string when the trigger is true.",
"type": "object"
}
},
"required": [
@@ -0,0 +1,62 @@
package agentexec
import (
"encoding/json"
"strings"
"testing"
)
// A capture tool sometimes needs a field only under a condition the same call
// carries: triage must name a workflow when it says a code change is needed,
// and must not be made to invent one when it says the opposite. A flat
// RequiredFields entry cannot say that, so the choice used to be between a
// meaningless forced answer and a silent default.
func TestValidateCaptureArgsConditionalRequirement(t *testing.T) {
spec := CaptureTool{
Name: "decide",
RequiredFields: []string{"needs_code_change"},
RequiredWhenTrue: map[string][]string{
"needs_code_change": {"workflow"},
},
}
tests := []struct {
name string
args string
wantOK bool
wantRejected string
}{
{
name: "trigger true and field present",
args: `{"needs_code_change": true, "workflow": "feasibility"}`,
wantOK: true,
},
{
name: "trigger true and field missing",
args: `{"needs_code_change": true}`,
wantRejected: "workflow",
},
{
name: "trigger true and field empty",
args: `{"needs_code_change": true, "workflow": " "}`,
wantRejected: "workflow",
},
{
name: "trigger false, field not required",
args: `{"needs_code_change": false}`,
wantOK: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rejection, ok := validateCaptureArgs(spec, json.RawMessage(tt.args))
if ok != tt.wantOK {
t.Fatalf("validateCaptureArgs ok = %v, want %v (rejection %q)", ok, tt.wantOK, rejection)
}
if tt.wantRejected != "" && !strings.Contains(rejection, tt.wantRejected) {
t.Fatalf("rejection %q does not name %q", rejection, tt.wantRejected)
}
})
}
}
+57 -10
View File
@@ -7,6 +7,7 @@ import (
"log/slog"
"maps"
"path/filepath"
"slices"
"strings"
"github.com/samcharles93/ai-sdk/agentloop"
@@ -283,32 +284,78 @@ func makeCaptureHandler(spec CaptureTool, captures map[string][]json.RawMessage)
// validateCaptureArgs checks that the capture tool arguments satisfy all
// constraints. Returns a rejection message and false on failure, or "" and
// true on success.
// true on success. Each constraint is its own check so adding one does not
// grow a single branching function.
func validateCaptureArgs(spec CaptureTool, value json.RawMessage) (string, bool) {
var object map[string]json.RawMessage
if err := json.Unmarshal(value, &object); err != nil {
return spec.Name + " rejected: arguments must be a JSON object", false //nolint:nilerr // the agent loop must see malformed tool arguments as feedback it can correct, not as a failed tool call
}
for _, check := range []func(CaptureTool, map[string]json.RawMessage) (string, bool){
checkRequiredFields,
checkNonEmptyStrings,
checkBooleanFields,
checkRequiredWhenTrue,
} {
if rejection, ok := check(spec, object); !ok {
return rejection, false
}
}
return "", true
}
func checkRequiredFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
for _, field := range spec.RequiredFields {
if _, ok := object[field]; !ok {
return fmt.Sprintf("%s rejected: %s is required", spec.Name, field), false
}
}
return "", true
}
func checkNonEmptyStrings(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
for _, field := range spec.NonEmptyStrings {
var text string
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &text) != nil || strings.TrimSpace(text) == "" {
return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument
}
}
for _, field := range spec.BooleanFields {
var val bool
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil {
return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false //nolint:nilerr // same: a rejection message lets the model retry with a valid argument
if !isNonEmptyString(object, field) {
return fmt.Sprintf("%s rejected: %s must be a non-empty string", spec.Name, field), false
}
}
return "", true
}
func checkBooleanFields(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
for _, field := range spec.BooleanFields {
var val bool
if raw, ok := object[field]; !ok || json.Unmarshal(raw, &val) != nil {
return fmt.Sprintf("%s rejected: %s must be a boolean", spec.Name, field), false
}
}
return "", true
}
// checkRequiredWhenTrue enforces the conditional requirements. Triggers are
// visited in sorted order so a call violating two of them names the same one
// every run, rather than a different one on each retry.
func checkRequiredWhenTrue(spec CaptureTool, object map[string]json.RawMessage) (string, bool) {
for _, trigger := range slices.Sorted(maps.Keys(spec.RequiredWhenTrue)) {
var on bool
if raw, ok := object[trigger]; !ok || json.Unmarshal(raw, &on) != nil || !on {
continue
}
for _, field := range spec.RequiredWhenTrue[trigger] {
if !isNonEmptyString(object, field) {
return fmt.Sprintf("%s rejected: %s is required when %s is true", spec.Name, field, trigger), false
}
}
}
return "", true
}
func isNonEmptyString(object map[string]json.RawMessage, field string) bool {
var text string
raw, ok := object[field]
return ok && json.Unmarshal(raw, &text) == nil && strings.TrimSpace(text) != ""
}
// scriptToolSet exposes run_go_script: interpreting a Yaegi Go script
// bundled with a skill (.agents/skills/<skill>/scripts/*.go) or living
// anywhere else in the workspace, and returning what it printed. This is
+9 -1
View File
@@ -52,7 +52,15 @@ type CaptureTool struct {
RequiredFields []string `json:"required_fields,omitempty"`
NonEmptyStrings []string `json:"non_empty_strings,omitempty"`
BooleanFields []string `json:"boolean_fields,omitempty"`
MaxCalls int `json:"max_calls,omitempty"`
// RequiredWhenTrue makes fields required only when a boolean field in
// the same call is true, keyed by that boolean's name. It exists because
// a flat RequiredFields entry cannot express "name a workflow, but only
// when you have said one is needed": the alternatives were forcing a
// meaningless answer on the other branch, or accepting the omission and
// defaulting it, which lets the model skip the decision entirely.
// A named field must also be a non-empty string when the trigger is true.
RequiredWhenTrue map[string][]string `json:"required_when_true,omitempty"`
MaxCalls int `json:"max_calls,omitempty"`
}
// Request is the complete, serializable input for one agent stage. The
+27
View File
@@ -51,6 +51,33 @@ func TestRouteLabelMatchStillWinsOverTriage(t *testing.T) {
}
}
// Forges namespace labels by convention, and a repo that does so was routing
// every issue through triage because no label ever matched a kind. Kind
// routing is the free, reliable signal Route's triage fallback assumes is
// already spent, so it has to survive the namespace.
func TestRouteMatchesNamespacedLabels(t *testing.T) {
reg := Registry{"tdd": TDD(), "feasibility": Feasibility(), "triage": Triage(), "implement": Implement()}
tests := []struct {
name string
labels string
want string
}{
{name: "namespaced feature", labels: "type::feature", want: "feasibility"},
{name: "namespaced bug", labels: "type::bug", want: "tdd"},
{name: "real issue label set", labels: "status::in_progress,priority::medium,type::feature", want: "feasibility"},
{name: "bare labels still route", labels: "bug", want: "tdd"},
{name: "no kind label still triages", labels: "priority::medium", want: "triage"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if wf := Route(&Task{Labels: tt.labels}, reg); wf.Name != tt.want {
t.Fatalf("Route(%q) = %q, want %q", tt.labels, wf.Name, tt.want)
}
})
}
}
// TestLoadKindWorkflowsYAMLOverridesDefaultBinding is the first slice of
// docs/prds/eda-playbook-engine.md: which workflow a Kind prefers becomes
// data, not a Go literal. A YAML file rebinding "bug" to "feasibility"
+40 -17
View File
@@ -9,8 +9,7 @@ import (
)
// triageWorkflowNames is the set of workflows triage may hand a task to.
// Unrecognized or missing classifier output falls back to "implement" --
// see triageDecideCaptureTools' schema description and Triage's OnResult.
// A name outside it is refused, not defaulted -- see Triage's OnResult.
var triageWorkflowNames = map[string]bool{
"implement": true,
"tdd": true,
@@ -44,18 +43,7 @@ func Triage() Workflow {
Role: "planner",
ReadOnly: true,
CaptureTools: triageDecideCaptureTools,
Mission: func(tc *TaskContext) string {
return fmt.Sprintf(
"Triage this %s on the repository %s: decide whether it needs a code "+
"change at all, and if so which workflow suits it best.\n\n"+
"%s\n\n"+
"Read only as much as you need to judge this -- a title/body that is "+
"purely conversational, administrative, or already resolved needs no code "+
"change. Then call the decide tool EXACTLY ONCE and afterwards call finish "+
"with status \"passed\".",
taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task),
)
},
Mission: triageMission,
OnResult: func(tc *TaskContext, res agentexec.Result) error {
calls := res.Captures["decide"]
if len(calls) != 1 {
@@ -85,9 +73,14 @@ func Triage() Workflow {
tc.Outcome = Outcome{Status: StatusMerged, Detail: "triaged: no code change needed -- " + captured.Reasons}
return nil
}
// No fallback. The decide tool already refuses a call that
// needs a code change and names no workflow, so reaching
// here with an unrecognised one means the model ignored
// its own enum; defaulting that to implement is how an
// unsettled capability used to reach the builder.
target := captured.Workflow
if !triageWorkflowNames[target] {
target = "implement"
return fmt.Errorf("triage chose workflow %q, which is not one of implement, tdd or feasibility", target)
}
tc.Task.Workflow = target
tc.Outcome = Outcome{
@@ -108,7 +101,7 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool {
"type": "object",
"properties": {
"needs_code_change": {"type": "boolean", "description": "false: nothing to build -- close/no-op. true: route to a workflow that builds something."},
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. Defaults to implement if omitted or unrecognized."},
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. feasibility: a new capability whose design is not settled, or a request too broad to scope -- it produces a design document for a human to approve before any code is written. tdd: a defect with an observable wrong behaviour that a test can reproduce first. implement: the change is well understood and its shape is already clear. Choose feasibility when no approved design exists and the request is not a defect; do not choose implement merely because nothing else obviously fits. Defaults to implement if omitted or unrecognized."},
"reasons": {"type": "string", "description": "The rationale, written for the human who filed the request."}
},
"required": ["needs_code_change", "reasons"]
@@ -116,6 +109,36 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool {
return []agentexec.CaptureTool{{
Name: "decide", Description: "Record the triage verdict. Call exactly once, before finish.",
Parameters: params, RequiredFields: []string{"needs_code_change", "reasons"},
NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, MaxCalls: 1,
NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"},
// Conditional, not flat: a task needing no code change has no
// workflow to name, and forcing one would be a meaningless answer.
RequiredWhenTrue: map[string][]string{"needs_code_change": {"workflow"}},
MaxCalls: 1,
}}
}
// triageMission is the classify stage's prompt. It is a named function so a
// test can assert the selection criteria are actually stated to the model,
// which is the whole of this stage's behaviour.
func triageMission(tc *TaskContext) string {
return fmt.Sprintf(
"Triage this %s on the repository %s: decide whether it needs a code "+
"change at all, and if so which workflow suits it best.\n\n"+
"%s\n\n"+
"Read only as much as you need to judge this -- a title/body that is "+
"purely conversational, administrative, or already resolved needs no code "+
"change.\n\n"+
"If it does need a change, the workflow is a real decision, not a "+
"formality. Check whether the repository already has a settled design "+
"for what is being asked (a design document, an approved plan, an "+
"existing implementation to extend). If it does not, and the request "+
"is not a defect, choose feasibility: it writes a design document for "+
"a human to approve, which is cheaper than an implementation built on "+
"a guess. Choose tdd when there is an observable wrong behaviour a "+
"test can reproduce first. Choose implement only when the change is "+
"well understood and its shape is already clear.\n\n"+
"Then call the decide tool EXACTLY ONCE and afterwards call finish "+
"with status \"passed\".",
taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task),
)
}
+82 -9
View File
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/agentexec"
@@ -140,10 +142,13 @@ func TestTriageRequeuesUnderChosenWorkflow(t *testing.T) {
}
}
// TestTriageDefaultsToImplementForUnrecognizedWorkflow confirms a missing
// or unrecognized workflow choice degrades to "implement" rather than
// producing an unroutable task.
func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
// Defaulting an unrecognised workflow to "implement" let the classifier skip
// the decision and send an unsettled capability straight to coding, which is
// the failure the selection criteria exist to prevent. Omission is caught
// earlier, by the decide tool's conditional requirement, so the model is
// told and retries; a value that is not in the enum at all means the model
// ignored its schema, and parking is the honest outcome.
func TestTriageRefusesAnUnrecognizedWorkflow(t *testing.T) {
runner := agentRunnerFunc(func(_ context.Context, _ string, req agentexec.Request, _ agentexec.ToolCallReporter) (agentexec.Result, error) {
return agentexec.Result{
Version: agentexec.ProtocolVersion,
@@ -151,7 +156,7 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
Attempt: req.Attempt,
Stage: req.Stage,
Status: agentexec.StatusPassed,
Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure, defaulting")},
Captures: map[string][]json.RawMessage{"decide": triageDecision(t, true, "not-a-real-workflow", "unsure")},
}, nil
})
tc := &TaskContext{
@@ -162,11 +167,32 @@ func TestTriageDefaultsToImplementForUnrecognizedWorkflow(t *testing.T) {
Log: slog.New(slog.DiscardHandler),
}
if err := classifyStage().Run(context.Background(), tc); err != nil {
t.Fatalf("classify stage = %v, want nil", err)
err := classifyStage().Run(context.Background(), tc)
if err == nil {
t.Fatal("classify stage accepted an unrecognised workflow")
}
if tc.Task.Workflow != "implement" {
t.Fatalf("Task.Workflow = %q, want %q", tc.Task.Workflow, "implement")
if !strings.Contains(err.Error(), "not-a-real-workflow") {
t.Fatalf("error %q does not name the rejected workflow", err)
}
if tc.Task.Workflow != "" {
t.Fatalf("Task.Workflow = %q, want it left unset", tc.Task.Workflow)
}
}
// The decide tool must demand a workflow exactly when one is needed, so an
// omission is a retryable rejection rather than a silent default, and the
// close-without-change branch is not made to invent an answer.
func TestTriageRequiresAWorkflowOnlyWhenACodeChangeIsNeeded(t *testing.T) {
tools := triageDecideCaptureTools(nil)
if len(tools) != 1 {
t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools))
}
required := tools[0].RequiredWhenTrue["needs_code_change"]
if !slices.Contains(required, "workflow") {
t.Fatalf("decide requires %v when needs_code_change is true, want it to include \"workflow\"", required)
}
if slices.Contains(tools[0].RequiredFields, "workflow") {
t.Error("decide requires workflow unconditionally; the close-without-change branch has no workflow to give")
}
}
@@ -188,3 +214,50 @@ func TestTriageRejectsMissingDecideCall(t *testing.T) {
t.Fatal("classify stage = nil, want an error when decide was never called")
}
}
// The classifier picks between three workflows, and until this was written
// neither the mission nor the decide schema said what any of them is for:
// the mission spent its guidance on needs_code_change and the schema said
// only "Which workflow fits best", with implement as the free default. Issue
// 813 (a new capability with no design doc) was classified implement on that
// basis and produced a half-wired slice, where feasibility would have
// produced the design doc this repo requires first.
//
// This asserts the criteria are stated, not how they are worded: each target
// must be named alongside the condition that selects it.
func TestTriageStatesItsWorkflowSelectionCriteria(t *testing.T) {
stage := classifyStage()
tc := &TaskContext{Task: &Task{Title: "t", Body: "b"}, Cfg: config.Config{}, Repo: config.Repo{Owner: "o", Name: "r"}}
tools := triageDecideCaptureTools(tc)
if len(tools) != 1 {
t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools))
}
schema := string(tools[0].Parameters)
for _, want := range []string{"feasibility", "tdd", "implement"} {
if !strings.Contains(schema, want) {
t.Errorf("decide schema never names the %q workflow", want)
}
}
// The criteria themselves. Each phrase is the discriminator a classifier
// needs to tell the three apart; without them the enum is unguided.
for _, want := range []string{"design", "reproduc"} {
if !strings.Contains(strings.ToLower(schema), want) {
t.Errorf("decide schema states no %q criterion for choosing a workflow", want)
}
}
if stage.Name != "classify" {
t.Fatalf("classify stage is named %q", stage.Name)
}
// The mission is what the model actually reads first, so the criteria
// have to be there too, not only in the tool schema.
mission := strings.ToLower(triageMission(tc))
for _, want := range []string{"feasibility", "tdd", "implement", "design", "reproduce"} {
if !strings.Contains(mission, want) {
t.Errorf("triage mission never mentions %q", want)
}
}
}
+21 -1
View File
@@ -95,9 +95,29 @@ func KindForLabels(labels []string) Kind {
func KindsForLabels(labels []string) []Kind {
var kinds []Kind
for _, label := range labels {
if kind, ok := labelKinds[strings.TrimSpace(label)]; ok {
if kind, ok := labelKinds[labelValue(label)]; ok {
kinds = append(kinds, kind)
}
}
return kinds
}
// labelValue reduces a forge label to the value labelKinds is keyed on.
//
// Forges namespace labels by convention ("type::feature", "priority::medium")
// and the namespace is the issue tracker's own taxonomy, not part of the
// routing vocabulary, so the value after the last "::" is what identifies a
// kind. Matching the bare string alone meant a repo that namespaces its
// labels -- which this one does -- never matched any kind, and every task
// fell through kind routing to the content-aware fallback in Route.
//
// The namespace itself is deliberately not checked: a closed three-value
// vocabulary (bug, feature, bootstrap) is specific enough that a label ending
// in one of them means that kind whatever the tracker files it under.
func labelValue(label string) string {
trimmed := strings.TrimSpace(label)
if i := strings.LastIndex(trimmed, "::"); i >= 0 {
return strings.TrimSpace(trimmed[i+len("::"):])
}
return trimmed
}
@@ -69,6 +69,11 @@ func TestKindForLabels(t *testing.T) {
{name: "first recognised wins", labels: []string{"feature", "bug"}, want: KindFeature},
{name: "unrecognised then recognised", labels: []string{"wontfix", "bootstrap"}, want: KindBootstrap},
{name: "trims whitespace", labels: []string{" bug "}, want: KindBug},
{name: "namespaced label", labels: []string{"type::feature"}, want: KindFeature},
{name: "namespaced, other namespaces ignored", labels: []string{"status::in_progress", "priority::medium", "type::feature"}, want: KindFeature},
{name: "namespaced and trimmed", labels: []string{" type::bug "}, want: KindBug},
{name: "namespace alone is not a kind", labels: []string{"type::"}, want: KindDefault},
{name: "nested namespaces use the last segment", labels: []string{"meta::type::bug"}, want: KindBug},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -90,6 +95,8 @@ func TestKindsForLabels(t *testing.T) {
{name: "single", labels: []string{"bug"}, want: []Kind{KindBug}},
{name: "preserves order", labels: []string{"bug", "feature"}, want: []Kind{KindBug, KindFeature}},
{name: "skips unrecognised", labels: []string{"wontfix", "bug", "triage", "feature"}, want: []Kind{KindBug, KindFeature}},
{name: "namespaced labels", labels: []string{"type::bug", "type::feature"}, want: []Kind{KindBug, KindFeature}},
{name: "mixed bare and namespaced", labels: []string{"status::open", "bug", "type::feature"}, want: []Kind{KindBug, KindFeature}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {