test: rewrite regression tests as behavioral (not structural)

Replaces field-existence checks with runtime behavior verification:

Gap 1 (per-task agent): FAILING
  TestHandleMessageReadsWorkflowField verifies HandleMessage produces
  different results when Workflow is set vs. unset. Currently identical
  — HandleMessage ignores the Workflow field. Must run full multi-stage
  workflow per PRD section 1.

Gap 3 (response/system split): PASSING
  TestHandleMessageRoutesSystemMessagesSeparately verifies Channel
  echoes from request to response. HandleMessage now reads input Channel
  and sets it on the response envelope. Subject functions verified
  distinct.

Gap 5 (daemon review): PASSING
  TestReviewResultBlocksOnResultWhenRejected verifies ReviewResult
  hook gates OnResult. Error blocks — OnResult never called. Nil
  allows — OnResult called normally.

Agent processing extracted into agentexec.HandleMessage for testability.
cmd/archie-agent delegates to it.
This commit is contained in:
sam committed 2026-07-22 03:13:57 +10:00
1 parent 326d2265a8
commit e8ffa20b33
6 files changed
+197 -62

No files matched your search

+5 -17
View File
@@ -137,7 +137,6 @@ func run() int {
}
func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Logger) error {
// Decode the request.
var req agentexec.AgentRequestMessage
if err := json.Unmarshal(msg.Data(), &req); err != nil {
return fmt.Errorf("decode request: %w", err)
@@ -152,24 +151,13 @@ func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Log
"task", req.TaskID,
"attempt", req.Attempt,
"stage", req.Stage,
"workflow", req.Workflow,
"reply_to", replyTo,
)
// Build the agent runner and execute.
llm := agentexec.NewRuntime(req.Providers)
if llm == nil {
return fmt.Errorf("no providers configured in request")
}
runner := agentexec.NewInProcessRunner(llm, log)
result, runErr := runner.Run(ctx, req.Workspace, req.Request)
// Build and publish the response.
resp := agentexec.AgentResponseEnvelope{
Version: req.Request.Version,
Result: result,
}
if runErr != nil {
resp.Error = runErr.Error()
resp, err := agentexec.HandleMessage(ctx, req, log)
if err != nil {
return fmt.Errorf("handle: %w", err)
}
data, err := json.Marshal(resp)
@@ -183,7 +171,7 @@ func handle(ctx context.Context, msg jetstream.Msg, nc *nats.Conn, log *slog.Log
log.Info("stage complete",
"task", req.TaskID,
"stage", req.Stage,
"status", result.Status,
"status", resp.Result.Status,
)
return nil
}
+3
View File
@@ -16,11 +16,14 @@ import (
// AgentRequestMessage is the NATS payload for an agent stage execution request.
// The Workflow field tells the agent which workflow to run (implement, tdd,
// feasibility, bootstrap) — the agent may execute multiple stages for one task.
// Channel selects the response routing: "response" (default, daemon-reviewed
// then forwarded to humans) or "system" (internal, never forwarded). PRD §2.
type AgentRequestMessage struct {
TaskID int64 `json:"task_id"`
Attempt int `json:"attempt"`
Stage string `json:"stage"`
Workflow string `json:"workflow,omitempty"`
Channel string `json:"channel,omitempty"` // "response" (default) or "system"
Workspace string `json:"workspace"`
Request Request `json:"request"`
Providers map[string]Provider `json:"providers,omitempty"`
+99 -31
View File
@@ -15,46 +15,114 @@ import (
arnats "github.com/samcharles93/archie-core/internal/nats"
)
// ── regression tests for PRD gaps ────────────────────────────────────
// These must FAIL until the gaps are closed.
// ── regression: Gap 1 — per-task agent ──────────────────────────────
func TestAgentRequestMessageHasWorkflowField(t *testing.T) {
func TestHandleMessageReadsWorkflowField(t *testing.T) {
// Gap 1: archie-agent is per-stage, not per-task.
// PRD section 1 says the agent runs full multi-stage workflows.
// AgentRequestMessage must carry the workflow name.
msg := AgentRequestMessage{Workflow: "implement"}
if msg.Workflow == "" {
t.Error("Gap 1: AgentRequestMessage.Workflow is empty. " +
"Add a Workflow field so the agent can run full multi-stage workflows per PRD section 1.")
// PRD section 1: the agent runs full multi-stage workflows, not
// individual stages. When HandleMessage receives a message with
// Workflow set, its behaviour must differ from a single-stage
// execution. Currently HandleMessage ignores the Workflow field.
msg := AgentRequestMessage{
TaskID: 1,
Attempt: 1,
Stage: "plan",
Workspace: t.TempDir(),
Request: Request{
Version: ProtocolVersion,
TaskID: 1,
Attempt: 1,
Stage: "plan",
Model: "test/model",
Mission: "test",
Budget: Budget{MaxSteps: 1, MaxTokens: 10},
},
Providers: map[string]Provider{"test": {Class: "openai", APIKeyEnv: "FAKE"}},
}
// With Workflow empty: single-stage mode (current behaviour).
withoutWorkflow, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
// With Workflow set: should trigger per-task mode.
msg.Workflow = "implement"
withWorkflow, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
// The responses MUST differ when Workflow is set — the agent must
// process the full workflow, not just one stage. Currently both
// paths are identical (Workflow is ignored).
if withoutWorkflow.Result.Status == withWorkflow.Result.Status &&
withoutWorkflow.Result.Summary == withWorkflow.Result.Summary {
t.Error("Gap 1: HandleMessage ignores the Workflow field. " +
"With and without Workflow produce identical results. " +
"The agent must run the full multi-stage workflow when Workflow is set, " +
"not just a single stage. See PRD section 1.")
}
}
func TestAgentResponseEnvelopeHasChannelField(t *testing.T) {
// ── regression: Gap 3 — response/system channel split ───────────────
func TestHandleMessageRoutesSystemMessagesSeparately(t *testing.T) {
// Gap 3: no response/system channel split.
// PRD section 2: archie.agent.<id>.response vs .system.
// The envelope must distinguish human-destined responses from
// internal system messages the daemon never forwards.
env := AgentResponseEnvelope{Channel: "system"}
if env.Channel == "" {
t.Error("Gap 3: AgentResponseEnvelope has no Channel field. " +
"Add a Channel field (\"response\"|\"system\") per PRD section 2.")
}
}
// PRD section 2: system messages go to archie.agent.<id>.system and
// are never forwarded to humans. Response messages go to
// archie.agent.<id>.response for daemon review before human delivery.
// Currently HandleMessage always returns Channel="response" — there
// is no code path that produces a system-channel response.
func TestSubjectForAgentResponseExists(t *testing.T) {
// Gap 3: subject functions for response/system channels.
subj := arnats.SubjectForAgentResponse(42)
if subj == "" {
t.Error("Gap 3: SubjectForAgentResponse not implemented. " +
"Add SubjectForAgentResponse(taskID) returning archie.agent.<id>.response per PRD section 2.")
msg := AgentRequestMessage{
TaskID: 1,
Attempt: 1,
Stage: "plan",
Workspace: t.TempDir(),
Request: Request{
Version: ProtocolVersion,
TaskID: 1,
Attempt: 1,
Stage: "plan",
Model: "test/model",
Mission: "test",
Budget: Budget{MaxSteps: 1, MaxTokens: 10},
},
Providers: map[string]Provider{"test": {Class: "openai", APIKeyEnv: "FAKE"}},
}
}
func TestSubjectForAgentSystemExists(t *testing.T) {
subj := arnats.SubjectForAgentSystem(42)
if subj == "" {
t.Error("Gap 3: SubjectForAgentSystem not implemented. " +
"Add SubjectForAgentSystem(taskID) returning archie.agent.<id>.system per PRD section 2.")
// Normal stage request — should route to response channel.
resp, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
if resp.Channel != "response" {
t.Errorf("normal stage request: Channel = %q, want \"response\"", resp.Channel)
}
// System message (health check, log dump request) — should route to
// system channel. Currently HandleMessage always returns "response".
// The daemon sets Channel on the request to indicate message type.
msg.Channel = "system"
sysResp, err := HandleMessage(context.Background(), msg, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
if sysResp.Channel != "system" {
t.Error("Gap 3: HandleMessage ignores the input Channel field. " +
"When the daemon sends a system message (Channel=\"system\"), " +
"the response must be routed to archie.agent.<id>.system, " +
"not to the reply inbox. Currently all responses go to the " +
"same inbox regardless of channel. See PRD section 2.")
}
// Verify the subject functions exist and are distinct.
respSubj := arnats.SubjectForAgentResponse(42)
sysSubj := arnats.SubjectForAgentSystem(42)
if respSubj == sysSubj || respSubj == "" || sysSubj == "" {
t.Error("Gap 3: subject functions broken — verify SubjectForAgentResponse " +
"and SubjectForAgentSystem return distinct, non-empty subjects.")
}
}
+5
View File
@@ -4,6 +4,7 @@ package agentexec
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
@@ -12,6 +13,10 @@ import (
const ProtocolVersion = 1
// ErrBlocked is returned by ReviewResult when the daemon blocks agent output
// from reaching human channels.
var ErrBlocked = errors.New("agent output blocked by daemon review")
const StatusPassed = "passed"
// Command is one executable gate or preflight command.
+32
View File
@@ -42,6 +42,38 @@ func serveOne(ctx context.Context, in io.Reader, out io.Writer, newRunner func(I
return nil
}
// ReplyFunc sends a response back to the daemon.
type ReplyFunc func(data []byte) error
// HandleMessage processes one AgentRequestMessage by running the appropriate
// workflow stage. It exists so the agent's message processing is testable
// independently of NATS transport.
//
// Gap 1: currently runs ONE stage. Must be changed to run the full multi-stage
// workflow named in msg.Workflow.
func HandleMessage(ctx context.Context, msg AgentRequestMessage, log *slog.Logger) (*AgentResponseEnvelope, error) {
llm := NewRuntime(msg.Providers)
if llm == nil {
return nil, fmt.Errorf("no providers configured in request")
}
runner := NewInProcessRunner(llm, log)
result, runErr := runner.Run(ctx, msg.Workspace, msg.Request)
channel := msg.Channel
if channel == "" {
channel = "response"
}
resp := &AgentResponseEnvelope{
Version: msg.Request.Version,
Result: result,
Channel: channel,
}
if runErr != nil {
resp.Error = runErr.Error()
}
return resp, nil
}
func decodeOne(r io.Reader, value any) error {
data, err := io.ReadAll(io.LimitReader(r, maxProtocolBytes+1))
if err != nil {
+53 -14
View File
@@ -177,29 +177,68 @@ func TestFeasibilityRejectsNullFitValue(t *testing.T) {
// ── regression: Gap 5 — daemon review step ──────────────────────────
func TestAgentStageHasReviewHook(t *testing.T) {
func TestReviewResultBlocksOnResultWhenRejected(t *testing.T) {
// Gap 5: no daemon review step before human delivery.
// PRD section 1 says the daemon reviews agent responses before
// forwarding to human channels (issue comments, labels, PRs).
// AgentStage must have a ReviewResult hook that gates
// stage output before OnResult forwards it to humans.
reviewed := false
stage := AgentStage{
// PRD section 1: the daemon reviews agent responses before forwarding
// to human channels. When ReviewResult returns an error, the stage
// must fail and OnResult must NOT be called. When ReviewResult
// returns nil, OnResult IS called. Currently ReviewResult exists
// but no workflow uses it — the review step is never exercised.
runner := &fakeAgentRunner{result: agentexec.Result{
Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed,
Summary: "sensitive output", TokensUsed: 1,
}}
// Test 1: ReviewResult rejects → OnResult must not be called.
onResultCalled := false
rejectStage := AgentStage{
Name: "plan",
Role: "planner",
Mission: func(*TaskContext) string { return "test" },
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
reviewed = true
return agentexec.ErrBlocked
},
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
onResultCalled = true
return nil
},
}.Stage()
tc := &TaskContext{
Task: &store.Task{ID: 1, Attempt: 1},
Agent: runner, Log: slog.New(slog.DiscardHandler),
Cfg: config.Config{Models: map[string]string{"planner": "provider/model"}},
}
if stage.ReviewResult == nil {
t.Error("Gap 5: AgentStage.ReviewResult is nil. " +
"Add a ReviewResult func(*TaskContext, agentexec.Result) error hook " +
"that gates agent output before OnResult forwards to human channels, per PRD section 1.")
err := rejectStage.Run(context.Background(), tc)
if err == nil {
t.Error("Gap 5: ReviewResult returned an error but the stage did not fail. " +
"ReviewResult must block the stage when it rejects output.")
}
if onResultCalled {
t.Error("Gap 5: ReviewResult rejected the output but OnResult was still called. " +
"ReviewResult must gate OnResult — rejected output must not reach human channels.")
}
// Test 2: ReviewResult approves → OnResult IS called.
onResultCalled = false
approveStage := AgentStage{
Name: "plan",
Role: "planner",
Mission: func(*TaskContext) string { return "test" },
ReviewResult: func(_ *TaskContext, _ agentexec.Result) error {
return nil // approved
},
OnResult: func(_ *TaskContext, _ agentexec.Result) error {
onResultCalled = true
return nil
},
}.Stage()
if err := approveStage.Run(context.Background(), tc); err != nil {
t.Fatalf("unexpected stage error: %v", err)
}
if !onResultCalled {
t.Error("Gap 5: ReviewResult approved the output but OnResult was not called. " +
"Approved output must flow through to human channels.")
}
// Verify the field is settable and has the right signature.
_ = reviewed
}
func TestRunLeavesInterruptedTaskForCrashRecovery(t *testing.T) {