feat(workflow): give triage criteria for choosing between workflows
The classify stage picked between implement, tdd and feasibility with no stated criteria: the mission spent its guidance on needs_code_change and the schema said only 'Which workflow fits best', with implement as the free default for an omitted or unrecognised answer. Issue 813 was classified implement on that basis. Both surfaces now state what each workflow is for and say not to pick implement merely because nothing else obviously fits. The mission moves to a named triageMission so a test can assert the criteria reach the model. Closes archie-core-s5ys
This commit is contained in:
1 parent
a73762d1e7
commit
4e3e48ca76
2 files changed
+76
-13
No files matched your search
@@ -44,18 +44,7 @@ func Triage() Workflow {
|
||||
Role: "planner",
|
||||
ReadOnly: true,
|
||||
CaptureTools: triageDecideCaptureTools,
|
||||
Mission: func(tc *TaskContext) string {
|
||||
return fmt.Sprintf(
|
||||
"Triage this %s on the repository %s: decide whether it needs a code "+
|
||||
"change at all, and if so which workflow suits it best.\n\n"+
|
||||
"%s\n\n"+
|
||||
"Read only as much as you need to judge this -- a title/body that is "+
|
||||
"purely conversational, administrative, or already resolved needs no code "+
|
||||
"change. Then call the decide tool EXACTLY ONCE and afterwards call finish "+
|
||||
"with status \"passed\".",
|
||||
taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task),
|
||||
)
|
||||
},
|
||||
Mission: triageMission,
|
||||
OnResult: func(tc *TaskContext, res agentexec.Result) error {
|
||||
calls := res.Captures["decide"]
|
||||
if len(calls) != 1 {
|
||||
@@ -108,7 +97,7 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"needs_code_change": {"type": "boolean", "description": "false: nothing to build -- close/no-op. true: route to a workflow that builds something."},
|
||||
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. Defaults to implement if omitted or unrecognized."},
|
||||
"workflow": {"type": "string", "enum": ["implement", "tdd", "feasibility"], "description": "Which workflow fits best, when needs_code_change is true. feasibility: a new capability whose design is not settled, or a request too broad to scope -- it produces a design document for a human to approve before any code is written. tdd: a defect with an observable wrong behaviour that a test can reproduce first. implement: the change is well understood and its shape is already clear. Choose feasibility when no approved design exists and the request is not a defect; do not choose implement merely because nothing else obviously fits. Defaults to implement if omitted or unrecognized."},
|
||||
"reasons": {"type": "string", "description": "The rationale, written for the human who filed the request."}
|
||||
},
|
||||
"required": ["needs_code_change", "reasons"]
|
||||
@@ -119,3 +108,29 @@ func triageDecideCaptureTools(*TaskContext) []agentexec.CaptureTool {
|
||||
NonEmptyStrings: []string{"reasons"}, BooleanFields: []string{"needs_code_change"}, MaxCalls: 1,
|
||||
}}
|
||||
}
|
||||
|
||||
// triageMission is the classify stage's prompt. It is a named function so a
|
||||
// test can assert the selection criteria are actually stated to the model,
|
||||
// which is the whole of this stage's behaviour.
|
||||
func triageMission(tc *TaskContext) string {
|
||||
return fmt.Sprintf(
|
||||
"Triage this %s on the repository %s: decide whether it needs a code "+
|
||||
"change at all, and if so which workflow suits it best.\n\n"+
|
||||
"%s\n\n"+
|
||||
"Read only as much as you need to judge this -- a title/body that is "+
|
||||
"purely conversational, administrative, or already resolved needs no code "+
|
||||
"change.\n\n"+
|
||||
"If it does need a change, the workflow is a real decision, not a "+
|
||||
"formality. Check whether the repository already has a settled design "+
|
||||
"for what is being asked (a design document, an approved plan, an "+
|
||||
"existing implementation to extend). If it does not, and the request "+
|
||||
"is not a defect, choose feasibility: it writes a design document for "+
|
||||
"a human to approve, which is cheaper than an implementation built on "+
|
||||
"a guess. Choose tdd when there is an observable wrong behaviour a "+
|
||||
"test can reproduce first. Choose implement only when the change is "+
|
||||
"well understood and its shape is already clear.\n\n"+
|
||||
"Then call the decide tool EXACTLY ONCE and afterwards call finish "+
|
||||
"with status \"passed\".",
|
||||
taskKind(tc.Task), tc.Repo.FullName(), taskPromptBlock(tc.Task),
|
||||
)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
@@ -188,3 +189,50 @@ func TestTriageRejectsMissingDecideCall(t *testing.T) {
|
||||
t.Fatal("classify stage = nil, want an error when decide was never called")
|
||||
}
|
||||
}
|
||||
|
||||
// The classifier picks between three workflows, and until this was written
|
||||
// neither the mission nor the decide schema said what any of them is for:
|
||||
// the mission spent its guidance on needs_code_change and the schema said
|
||||
// only "Which workflow fits best", with implement as the free default. Issue
|
||||
// 813 (a new capability with no design doc) was classified implement on that
|
||||
// basis and produced a half-wired slice, where feasibility would have
|
||||
// produced the design doc this repo requires first.
|
||||
//
|
||||
// This asserts the criteria are stated, not how they are worded: each target
|
||||
// must be named alongside the condition that selects it.
|
||||
func TestTriageStatesItsWorkflowSelectionCriteria(t *testing.T) {
|
||||
stage := classifyStage()
|
||||
tc := &TaskContext{Task: &Task{Title: "t", Body: "b"}, Cfg: config.Config{}, Repo: config.Repo{Owner: "o", Name: "r"}}
|
||||
|
||||
tools := triageDecideCaptureTools(tc)
|
||||
if len(tools) != 1 {
|
||||
t.Fatalf("triageDecideCaptureTools returned %d tools, want 1", len(tools))
|
||||
}
|
||||
schema := string(tools[0].Parameters)
|
||||
|
||||
for _, want := range []string{"feasibility", "tdd", "implement"} {
|
||||
if !strings.Contains(schema, want) {
|
||||
t.Errorf("decide schema never names the %q workflow", want)
|
||||
}
|
||||
}
|
||||
|
||||
// The criteria themselves. Each phrase is the discriminator a classifier
|
||||
// needs to tell the three apart; without them the enum is unguided.
|
||||
for _, want := range []string{"design", "reproduc"} {
|
||||
if !strings.Contains(strings.ToLower(schema), want) {
|
||||
t.Errorf("decide schema states no %q criterion for choosing a workflow", want)
|
||||
}
|
||||
}
|
||||
if stage.Name != "classify" {
|
||||
t.Fatalf("classify stage is named %q", stage.Name)
|
||||
}
|
||||
|
||||
// The mission is what the model actually reads first, so the criteria
|
||||
// have to be there too, not only in the tool schema.
|
||||
mission := strings.ToLower(triageMission(tc))
|
||||
for _, want := range []string{"feasibility", "tdd", "implement", "design", "reproduce"} {
|
||||
if !strings.Contains(mission, want) {
|
||||
t.Errorf("triage mission never mentions %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user