test: rewrite regression tests as behavioral (not structural)
Replaces field-existence checks with runtime behavior verification: Gap 1 (per-task agent): FAILING TestHandleMessageReadsWorkflowField verifies HandleMessage produces different results when Workflow is set vs. unset. Currently identical — HandleMessage ignores the Workflow field. Must run full multi-stage workflow per PRD section 1. Gap 3 (response/system split): PASSING TestHandleMessageRoutesSystemMessagesSeparately verifies Channel echoes from request to response. HandleMessage now reads input Channel and sets it on the response envelope. Subject functions verified distinct. Gap 5 (daemon review): PASSING TestReviewResultBlocksOnResultWhenRejected verifies ReviewResult hook gates OnResult. Error blocks — OnResult never called. Nil allows — OnResult called normally. Agent processing extracted into agentexec.HandleMessage for testability. cmd/archie-agent delegates to it.
This commit is contained in:
1 parent
326d2265a8
commit
e8ffa20b33
6 files changed
+197
-62
No files matched your search
@@ -137,7 +137,6 @@ func run() int {
|
||||
}
|
||||
|
||||
func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Logger) error {
|
||||
// Decode the request.
|
||||
var req agentexec.AgentRequestMessage
|
||||
if err := json.Unmarshal(msg.Data(), &req); err != nil {
|
||||
return fmt.Errorf("decode request: %w", err)
|
||||
@@ -152,24 +151,13 @@ func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Log
|
||||
"task", req.TaskID,
|
||||
"attempt", req.Attempt,
|
||||
"stage", req.Stage,
|
||||
"workflow", req.Workflow,
|
||||
"reply_to", replyTo,
|
||||
)
|
||||
|
||||
// Build the agent runner and execute.
|
||||
llm := agentexec.NewRuntime(req.Providers)
|
||||
if llm == nil {
|
||||
return fmt.Errorf("no providers configured in request")
|
||||
}
|
||||
runner := agentexec.NewInProcessRunner(llm, log)
|
||||
result, runErr := runner.Run(ctx, req.Workspace, req.Request)
|
||||
|
||||
// Build and publish the response.
|
||||
resp := agentexec.AgentResponseEnvelope{
|
||||
Version: req.Request.Version,
|
||||
Result: result,
|
||||
}
|
||||
if runErr != nil {
|
||||
resp.Error = runErr.Error()
|
||||
resp, err := agentexec.HandleMessage(ctx, req, log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("handle: %w", err)
|
||||
}
|
||||
|
||||
data, err := json.Marshal(resp)
|
||||
@@ -183,7 +171,7 @@ func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Log
|
||||
log.Info("stage complete",
|
||||
"task", req.TaskID,
|
||||
"stage", req.Stage,
|
||||
"status", result.Status,
|
||||
"status", resp.Result.Status,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
@@ -16,11 +16,14 @@ import (
|
||||
// AgentRequestMessage is the NATS payload for an agent stage execution request.
|
||||
// The Workflow field tells the agent which workflow to run (implement, tdd,
|
||||
// feasibility, bootstrap) — the agent may execute multiple stages for one task.
|
||||
// Channel selects the response routing: "response" (default, daemon-reviewed
|
||||
// then forwarded to humans) or "system" (internal, never forwarded). PRD §2.
|
||||
type AgentRequestMessage struct {
|
||||
TaskID int64 `json:"task_id"`
|
||||
Attempt int `json:"attempt"`
|
||||
Stage string `json:"stage"`
|
||||
Workflow string `json:"workflow,omitempty"`
|
||||
Channel string `json:"channel,omitempty"` // "response" (default) or "system"
|
||||
Workspace string `json:"workspace"`
|
||||
Request Request `json:"request"`
|
||||
Providers map[string]Provider `json:"providers,omitempty"`
|
||||
|
||||
@@ -15,46 +15,114 @@ import (
|
||||
arnats "github.com/samcharles93/archie-core/internal/nats"
|
||||
)
|
||||
|
||||
// ── regression tests for PRD gaps ────────────────────────────────────
|
||||
// These must FAIL until the gaps are closed.
|
||||
// ── regression: Gap 1 — per-task agent ──────────────────────────────
|
||||
|
||||
func TestAgentRequestMessageHasWorkflowField(t *testing.T) {
|
||||
func TestHandleMessageReadsWorkflowField(t *testing.T) {
|
||||
// Gap 1: archie-agent is per-stage, not per-task.
|
||||
// PRD section 1 says the agent runs full multi-stage workflows.
|
||||
// AgentRequestMessage must carry the workflow name.
|
||||
msg := AgentRequestMessage{Workflow: "implement"}
|
||||
if msg.Workflow == "" {
|
||||
t.Error("Gap 1: AgentRequestMessage.Workflow is empty. " +
|
||||
"Add a Workflow field so the agent can run full multi-stage workflows per PRD section 1.")
|
||||
// PRD section 1: the agent runs full multi-stage workflows, not
|
||||
// individual stages. When HandleMessage receives a message with
|
||||
// Workflow set, its behaviour must differ from a single-stage
|
||||
// execution. Currently HandleMessage ignores the Workflow field.
|
||||
msg := AgentRequestMessage{
|
||||
TaskID: 1,
|
||||
Attempt: 1,
|
||||
Stage: "plan",
|
||||
Workspace: t.TempDir(),
|
||||
Request: Request{
|
||||
Version: ProtocolVersion,
|
||||
TaskID: 1,
|
||||
Attempt: 1,
|
||||
Stage: "plan",
|
||||
Model: "test/model",
|
||||
Mission: "test",
|
||||
Budget: Budget{MaxSteps: 1, MaxTokens: 10},
|
||||
},
|
||||
Providers: map[string]Provider{"test": {Class: "openai", APIKeyEnv: "FAKE"}},
|
||||
}
|
||||
|
||||
// With Workflow empty: single-stage mode (current behaviour).
|
||||
withoutWorkflow, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// With Workflow set: should trigger per-task mode.
|
||||
msg.Workflow = "implement"
|
||||
withWorkflow, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The responses MUST differ when Workflow is set — the agent must
|
||||
// process the full workflow, not just one stage. Currently both
|
||||
// paths are identical (Workflow is ignored).
|
||||
if withoutWorkflow.Result.Status == withWorkflow.Result.Status &&
|
||||
withoutWorkflow.Result.Summary == withWorkflow.Result.Summary {
|
||||
t.Error("Gap 1: HandleMessage ignores the Workflow field. " +
|
||||
"With and without Workflow produce identical results. " +
|
||||
"The agent must run the full multi-stage workflow when Workflow is set, " +
|
||||
"not just a single stage. See PRD section 1.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentResponseEnvelopeHasChannelField(t *testing.T) {
|
||||
// ── regression: Gap 3 — response/system channel split ───────────────
|
||||
|
||||
func TestHandleMessageRoutesSystemMessagesSeparately(t *testing.T) {
|
||||
// Gap 3: no response/system channel split.
|
||||
// PRD section 2: archie.agent.<id>.response vs .system.
|
||||
// The envelope must distinguish human-destined responses from
|
||||
// internal system messages the daemon never forwards.
|
||||
env := AgentResponseEnvelope{Channel: "system"}
|
||||
if env.Channel == "" {
|
||||
t.Error("Gap 3: AgentResponseEnvelope has no Channel field. " +
|
||||
"Add a Channel field (\"response\"|\"system\") per PRD section 2.")
|
||||
}
|
||||
}
|
||||
// PRD section 2: system messages go to archie.agent.<id>.system and
|
||||
// are never forwarded to humans. Response messages go to
|
||||
// archie.agent.<id>.response for daemon review before human delivery.
|
||||
// Currently HandleMessage always returns Channel="response" — there
|
||||
// is no code path that produces a system-channel response.
|
||||
|
||||
func TestSubjectForAgentResponseExists(t *testing.T) {
|
||||
// Gap 3: subject functions for response/system channels.
|
||||
subj := arnats.SubjectForAgentResponse(42)
|
||||
if subj == "" {
|
||||
t.Error("Gap 3: SubjectForAgentResponse not implemented. " +
|
||||
"Add SubjectForAgentResponse(taskID) returning archie.agent.<id>.response per PRD section 2.")
|
||||
msg := AgentRequestMessage{
|
||||
TaskID: 1,
|
||||
Attempt: 1,
|
||||
Stage: "plan",
|
||||
Workspace: t.TempDir(),
|
||||
Request: Request{
|
||||
Version: ProtocolVersion,
|
||||
TaskID: 1,
|
||||
Attempt: 1,
|
||||
Stage: "plan",
|
||||
Model: "test/model",
|
||||
Mission: "test",
|
||||
Budget: Budget{MaxSteps: 1, MaxTokens: 10},
|
||||
},
|
||||
Providers: map[string]Provider{"test": {Class: "openai", APIKeyEnv: "FAKE"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubjectForAgentSystemExists(t *testing.T) {
|
||||
subj := arnats.SubjectForAgentSystem(42)
|
||||
if subj == "" {
|
||||
t.Error("Gap 3: SubjectForAgentSystem not implemented. " +
|
||||
"Add SubjectForAgentSystem(taskID) returning archie.agent.<id>.system per PRD section 2.")
|
||||
// Normal stage request — should route to response channel.
|
||||
resp, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.Channel != "response" {
|
||||
t.Errorf("normal stage request: Channel = %q, want \"response\"", resp.Channel)
|
||||
}
|
||||
|
||||
// System message (health check, log dump request) — should route to
|
||||
// system channel. Currently HandleMessage always returns "response".
|
||||
// The daemon sets Channel on the request to indicate message type.
|
||||
msg.Channel = "system"
|
||||
sysResp, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sysResp.Channel != "system" {
|
||||
t.Error("Gap 3: HandleMessage ignores the input Channel field. " +
|
||||
"When the daemon sends a system message (Channel=\"system\"), " +
|
||||
"the response must be routed to archie.agent.<id>.system, " +
|
||||
"not to the reply inbox. Currently all responses go to the " +
|
||||
"same inbox regardless of channel. See PRD section 2.")
|
||||
}
|
||||
|
||||
// Verify the subject functions exist and are distinct.
|
||||
respSubj := arnats.SubjectForAgentResponse(42)
|
||||
sysSubj := arnats.SubjectForAgentSystem(42)
|
||||
if respSubj == sysSubj || respSubj == "" || sysSubj == "" {
|
||||
t.Error("Gap 3: subject functions broken — verify SubjectForAgentResponse " +
|
||||
"and SubjectForAgentSystem return distinct, non-empty subjects.")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ package agentexec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -12,6 +13,10 @@ import (
|
||||
|
||||
const ProtocolVersion = 1
|
||||
|
||||
// ErrBlocked is returned by ReviewResult when the daemon blocks agent output
|
||||
// from reaching human channels.
|
||||
var ErrBlocked = errors.New("agent output blocked by daemon review")
|
||||
|
||||
const StatusPassed = "passed"
|
||||
|
||||
// Command is one executable gate or preflight command.
|
||||
|
||||
@@ -42,6 +42,38 @@ func serveOne(ctx context.Context, in io.Reader, out io.Writer, newRunner func(I
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReplyFunc sends a response back to the daemon.
|
||||
type ReplyFunc func(data []byte) error
|
||||
|
||||
// HandleMessage processes one AgentRequestMessage by running the appropriate
|
||||
// workflow stage. It exists so the agent's message processing is testable
|
||||
// independently of NATS transport.
|
||||
//
|
||||
// Gap 1: currently runs ONE stage. Must be changed to run the full multi-stage
|
||||
// workflow named in msg.Workflow.
|
||||
func HandleMessage(ctx context.Context, msg AgentRequestMessage, log *slog.Logger) (*AgentResponseEnvelope, error) {
|
||||
llm := NewRuntime(msg.Providers)
|
||||
if llm == nil {
|
||||
return nil, fmt.Errorf("no providers configured in request")
|
||||
}
|
||||
runner := NewInProcessRunner(llm, log)
|
||||
result, runErr := runner.Run(ctx, msg.Workspace, msg.Request)
|
||||
|
||||
channel := msg.Channel
|
||||
if channel == "" {
|
||||
channel = "response"
|
||||
}
|
||||
resp := &AgentResponseEnvelope{
|
||||
Version: msg.Request.Version,
|
||||
Result: result,
|
||||
Channel: channel,
|
||||
}
|
||||
if runErr != nil {
|
||||
resp.Error = runErr.Error()
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func decodeOne(r io.Reader, value any) error {
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxProtocolBytes+1))
|
||||
if err != nil {
|
||||
|
||||
@@ -177,29 +177,68 @@ func TestFeasibilityRejectsNullFitValue(t *testing.T) {
|
||||
|
||||
// ── regression: Gap 5 — daemon review step ──────────────────────────
|
||||
|
||||
func TestAgentStageHasReviewHook(t *testing.T) {
|
||||
func TestReviewResultBlocksOnResultWhenRejected(t *testing.T) {
|
||||
// Gap 5: no daemon review step before human delivery.
|
||||
// PRD section 1 says the daemon reviews agent responses before
|
||||
// forwarding to human channels (issue comments, labels, PRs).
|
||||
// AgentStage must have a ReviewResult hook that gates
|
||||
// stage output before OnResult forwards it to humans.
|
||||
reviewed := false
|
||||
stage := AgentStage{
|
||||
// PRD section 1: the daemon reviews agent responses before forwarding
|
||||
// to human channels. When ReviewResult returns an error, the stage
|
||||
// must fail and OnResult must NOT be called. When ReviewResult
|
||||
// returns nil, OnResult IS called. Currently ReviewResult exists
|
||||
// but no workflow uses it — the review step is never exercised.
|
||||
runner := &fakeAgentRunner{result: agentexec.Result{
|
||||
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
|
||||
Summary: "sensitive output", TokensUsed: 1,
|
||||
}}
|
||||
|
||||
// Test 1: ReviewResult rejects → OnResult must not be called.
|
||||
onResultCalled := false
|
||||
rejectStage := AgentStage{
|
||||
Name: "plan",
|
||||
Role: "planner",
|
||||
Mission: func(*TaskContext) string { return "test" },
|
||||
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
|
||||
reviewed = true
|
||||
return agentexec.ErrBlocked
|
||||
},
|
||||
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
|
||||
onResultCalled = true
|
||||
return nil
|
||||
},
|
||||
}.Stage()
|
||||
tc := &TaskContext{
|
||||
Task: &store.Task{ID: 1, Attempt: 1},
|
||||
Agent: runner, Log: slog.New(slog.DiscardHandler),
|
||||
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
|
||||
}
|
||||
if stage.ReviewResult == nil {
|
||||
t.Error("Gap 5: AgentStage.ReviewResult is nil. " +
|
||||
"Add a ReviewResult func(*TaskContext, agentexec.Result) error hook " +
|
||||
"that gates agent output before OnResult forwards to human channels, per PRD section 1.")
|
||||
err := rejectStage.Run(context.Background(), tc)
|
||||
if err == nil {
|
||||
t.Error("Gap 5: ReviewResult returned an error but the stage did not fail. " +
|
||||
"ReviewResult must block the stage when it rejects output.")
|
||||
}
|
||||
if onResultCalled {
|
||||
t.Error("Gap 5: ReviewResult rejected the output but OnResult was still called. " +
|
||||
"ReviewResult must gate OnResult — rejected output must not reach human channels.")
|
||||
}
|
||||
|
||||
// Test 2: ReviewResult approves → OnResult IS called.
|
||||
onResultCalled = false
|
||||
approveStage := AgentStage{
|
||||
Name: "plan",
|
||||
Role: "planner",
|
||||
Mission: func(*TaskContext) string { return "test" },
|
||||
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
|
||||
return nil // approved
|
||||
},
|
||||
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
|
||||
onResultCalled = true
|
||||
return nil
|
||||
},
|
||||
}.Stage()
|
||||
if err := approveStage.Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("unexpected stage error: %v", err)
|
||||
}
|
||||
if !onResultCalled {
|
||||
t.Error("Gap 5: ReviewResult approved the output but OnResult was not called. " +
|
||||
"Approved output must flow through to human channels.")
|
||||
}
|
||||
// Verify the field is settable and has the right signature.
|
||||
_ = reviewed
|
||||
}
|
||||
|
||||
func TestRunLeavesInterruptedTaskForCrashRecovery(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user