diff --git a/internal/domain/workflow/triage.go b/internal/domain/workflow/triage.go index 3a95d4f40..9bddc9e79 100644 --- a/internal/domain/workflow/triage.go +++ b/internal/domain/workflow/triage.go @@ -44,18 +44,7 @@ func Triage() Workflow { Role: "planner", ReadOnly: true, CaptureTools: triageDecideCaptureTools, - Mission: func(tc *TaskContext) string { - return fmt.Sprintf( - "Triage this %s on the repository %s: decide whether it needs a code "+ - "change at all, and if so which workflow suits it best.\n\n"+ - "%s\n\n"+ - "Read only as much as you need to judge this -- a title/body that is "+ - "purely conversational, administrative, or already resolved needs no code "+ - "change. Then call the decide tool EXACTLY ONCE and afterwards call finish "+ - "with status \"passed\".", - taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task), - ) - }, + Mission: triageMission, OnResult: func(tc *TaskContext, res agentexec.Result) error { calls := res.Captures["decide"] if len(calls) != 1 { @@ -108,7 +97,7 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool { "type": "object", "properties": { "needs_code_change": {"type": "boolean", "description": "false: nothing to build -- close/no-op. true: route to a workflow that builds something."}, - "workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. Defaults to implement if omitted or unrecognized."}, + "workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. feasibility: a new capability whose design is not settled, or a request too broad to scope -- it produces a design document for a human to approve before any code is written. tdd: a defect with an observable wrong behaviour that a test can reproduce first. implement: the change is well understood and its shape is already clear. Choose feasibility when no approved design exists and the request is not a defect; do not choose implement merely because nothing else obviously fits. Defaults to implement if omitted or unrecognized."}, "reasons": {"type": "string", "description": "The rationale, written for the human who filed the request."} }, "required": ["needs_code_change", "reasons"] @@ -119,3 +108,29 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool { NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, MaxCalls: 1, }} } + +// triageMission is the classify stage's prompt. It is a named function so a +// test can assert the selection criteria are actually stated to the model, +// which is the whole of this stage's behaviour. +func triageMission(tc *TaskContext) string { + return fmt.Sprintf( + "Triage this %s on the repository %s: decide whether it needs a code "+ + "change at all, and if so which workflow suits it best.\n\n"+ + "%s\n\n"+ + "Read only as much as you need to judge this -- a title/body that is "+ + "purely conversational, administrative, or already resolved needs no code "+ + "change.\n\n"+ + "If it does need a change, the workflow is a real decision, not a "+ + "formality. Check whether the repository already has a settled design "+ + "for what is being asked (a design document, an approved plan, an "+ + "existing implementation to extend). If it does not, and the request "+ + "is not a defect, choose feasibility: it writes a design document for "+ + "a human to approve, which is cheaper than an implementation built on "+ + "a guess. Choose tdd when there is an observable wrong behaviour a "+ + "test can reproduce first. Choose implement only when the change is "+ + "well understood and its shape is already clear.\n\n"+ + "Then call the decide tool EXACTLY ONCE and afterwards call finish "+ + "with status \"passed\".", + taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task), + ) +} diff --git a/internal/domain/workflow/triage_test.go b/internal/domain/workflow/triage_test.go index ab014310d..1e79d6f3f 100644 --- a/internal/domain/workflow/triage_test.go +++ b/internal/domain/workflow/triage_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "log/slog" + "strings" "testing" "github.com/samcharles93/archie-core/internal/agentexec" @@ -188,3 +189,50 @@ func TestTriageRejectsMissingDecideCall(t *testing.T) { t.Fatal("classify stage = nil, want an error when decide was never called") } } + +// The classifier picks between three workflows, and until this was written +// neither the mission nor the decide schema said what any of them is for: +// the mission spent its guidance on needs_code_change and the schema said +// only "Which workflow fits best", with implement as the free default. Issue +// 813 (a new capability with no design doc) was classified implement on that +// basis and produced a half-wired slice, where feasibility would have +// produced the design doc this repo requires first. +// +// This asserts the criteria are stated, not how they are worded: each target +// must be named alongside the condition that selects it. +func TestTriageStatesItsWorkflowSelectionCriteria(t *testing.T) { + stage := classifyStage() + tc := &TaskContext{Task: &Task{Title: "t", Body: "b"}, Cfg: config.Config{}, Repo: config.Repo{Owner: "o", Name: "r"}} + + tools := triageDecideCaptureTools(tc) + if len(tools) != 1 { + t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools)) + } + schema := string(tools[0].Parameters) + + for _, want := range []string{"feasibility", "tdd", "implement"} { + if !strings.Contains(schema, want) { + t.Errorf("decide schema never names the %q workflow", want) + } + } + + // The criteria themselves. Each phrase is the discriminator a classifier + // needs to tell the three apart; without them the enum is unguided. + for _, want := range []string{"design", "reproduc"} { + if !strings.Contains(strings.ToLower(schema), want) { + t.Errorf("decide schema states no %q criterion for choosing a workflow", want) + } + } + if stage.Name != "classify" { + t.Fatalf("classify stage is named %q", stage.Name) + } + + // The mission is what the model actually reads first, so the criteria + // have to be there too, not only in the tool schema. + mission := strings.ToLower(triageMission(tc)) + for _, want := range []string{"feasibility", "tdd", "implement", "design", "reproduce"} { + if !strings.Contains(mission, want) { + t.Errorf("triage mission never mentions %q", want) + } + } +}