From e9a79ec3ed2c2cc76a4871e214f1ea4e034e12be Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 2 Oct 2026 09:46:16 +1000 Subject: [PATCH] fix(workflow): park environment gate failures without dispatching a builder The baseline stage treated every red gate as a pre-existing code defect and dispatched a TDD repair agent. A failure the environment caused -- no Docker daemon for the test harness, or a gate command that is not installed -- cannot be fixed from the worktree, so the run spent a full agent budget and then parked. A provider error during that repair run (observed: HTTP 402) replaced the gate cause in the park reason entirely. Classify at the producer: gateEnvironmentCause names a failure whose command could not start (*exec.Error) or whose output names unavailable host infrastructure (the Docker shapes from archie-core-71mq). That failure parks with the cause and the gate output and starts no builder. A real code failure still dispatches exactly one repair run, and every post-dispatch failure now carries the gate output alongside the agent/provider error instead of masking it. Refs archie-core-rwy6. --- internal/domain/workflow/gate_environment.go | 87 ++++++++ .../domain/workflow/gate_environment_test.go | 210 ++++++++++++++++++ internal/domain/workflow/implement.go | 140 ++++++------ 3 files changed, 375 insertions(+), 62 deletions(-) create mode 100644 internal/domain/workflow/gate_environment.go create mode 100644 internal/domain/workflow/gate_environment_test.go diff --git a/internal/domain/workflow/gate_environment.go b/internal/domain/workflow/gate_environment.go new file mode 100644 index 000000000..56dccfe1b --- /dev/null +++ b/internal/domain/workflow/gate_environment.go @@ -0,0 +1,87 @@ +package workflow + +import ( + "errors" + "fmt" + "os/exec" + "strings" +) + +// gateEnvironmentCause reports why a failed gate command is an environment +// failure the baseline repair agent cannot fix, or "" when the failure is an +// ordinary code failure the agent can act on. +// +// Only two signals are treated as reliable enough to act on: +// +// - the command could not be started at all. os/exec reports a missing or +// non-executable binary as *exec.Error before the gate runs a line of its +// own, and no source change in the repository can make a tool that is not +// installed appear. +// - the combined output names host infrastructure the gate needs and does +// not have, such as a Docker daemon the test harness spawns containers on. +// These are the phrasings the tools themselves emit when their dependency +// is absent (archie-core-71mq), not an inference drawn from the exit +// status. +// +// The second signal is deliberately a closed, literal set: a marker is added +// with the incident that produced it, never by inference. A false negative +// only spends the repair budget a code failure would have spent anyway, but a +// false positive would park a task the builder could have fixed. +func gateEnvironmentCause(out string, runErr error) string { + if execErr, ok := errors.AsType[*exec.Error](runErr); ok { + return "the gate command could not be started (" + execErr.Error() + ")" + } + lower := strings.ToLower(out) + for _, m := range gateEnvironmentMarkers { + if strings.Contains(lower, m.marker) { + return m.cause + } + } + return "" +} + +// gateEnvironmentMarkers pairs an output fragment a missing host resource +// produces with the cause it names. Markers are matched case-insensitively +// against the gate's combined output. +var gateEnvironmentMarkers = []struct { + marker string + cause string +}{ + { + marker: "is docker running?", + cause: "the Docker daemon required by the test harness is unavailable", + }, + { + marker: "cannot connect to the docker daemon", + cause: "the Docker daemon is unavailable", + }, + { + marker: "rootless docker not found", + cause: "the Docker daemon is unavailable", + }, + { + marker: "failed to create docker provider", + cause: "the Docker daemon is unavailable", + }, + { + marker: "docker: command not found", + cause: "the docker CLI is not installed", + }, +} + +// baselineEnvironmentError is the park reason when a gate failed for an +// environment cause: it names the cause and keeps the gate output an operator +// needs to confirm it. The stage returns this without dispatching a builder. +func baselineEnvironmentError(argv []string, out, cause string) error { + return fmt.Errorf("baseline red -- %s cannot run in this environment: %s\n\ngate output:\n%s", + strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes)) +} + +// baselineFixFailedError is the park reason for every failure after the +// baseline repair agent was dispatched. A failure that names only the agent -- +// a provider 402, say -- and drops the gate output leaves the operator with no +// way to see why the gate was red in the first place (archie-core-rwy6). +func baselineFixFailedError(argv []string, out, cause string) error { + return fmt.Errorf("baseline red -- %s fails and the baseline-fix builder did not repair it: %s\n\ngate output:\n%s", + strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes)) +} diff --git a/internal/domain/workflow/gate_environment_test.go b/internal/domain/workflow/gate_environment_test.go new file mode 100644 index 000000000..bebaea2da --- /dev/null +++ b/internal/domain/workflow/gate_environment_test.go @@ -0,0 +1,210 @@ +package workflow + +import ( + "context" + "errors" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/samcharles93/archie-core/internal/agentexec" + "github.com/samcharles93/archie-core/internal/config" +) + +// gateScript writes an executable shell script running `body` and returns +// the argv that invokes it. +func gateScript(t *testing.T, dir, body string) []string { + t.Helper() + path := filepath.Join(dir, "gate.sh") + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + return []string{"sh", path} +} + +// TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures pins the +// classification the baseline stage acts on. It must recognise the shapes a +// missing host resource produces (the command could not start at all, or a +// tool says its dependency is absent) and must not mistake an ordinary code +// failure for one of them -- a false positive parks a task the builder could +// have fixed. +func TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures(t *testing.T) { + failingExit := errors.New("exit status 1") + missingBinary := &exec.Error{Name: "go", Err: exec.ErrNotFound} + + tests := []struct { + name string + out string + err error + want bool + }{ + { + name: "pgtest cannot reach the docker daemon", + out: "pgtest: start postgres:18 (is Docker running?): " + + "rootless Docker not found, failed to create Docker provider\nFAIL\tinternal/webui", + err: failingExit, + want: true, + }, + { + name: "testcontainers cannot connect to the daemon", + out: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?", + err: failingExit, + want: true, + }, + { + name: "the gate binary is not installed", + err: missingBinary, + want: true, + }, + { + name: "an ordinary test failure is a code failure", + out: "--- FAIL: TestBroken (0.00s)\n broken_test.go:5: want 1, got 2\nFAIL\tpkg/x", + err: failingExit, + want: false, + }, + { + name: "a compile error is a code failure", + out: "# pkg/x\n./x.go:3:2: undefined: missing", + err: failingExit, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cause := gateEnvironmentCause(tt.out, tt.err) + if got := cause != ""; got != tt.want { + t.Fatalf("gateEnvironmentCause() = %q, want environmental=%v", cause, tt.want) + } + }) + } +} + +// TestStageBaselineGateParksEnvironmentFailureWithoutDispatch is acceptance +// (a): a gate that failed because the environment lacks a resource parks with +// that cause and never starts the baseline-fix builder -- nothing in the +// worktree can fix a missing Docker daemon, so dispatching one only burns the +// run budget. +func TestStageBaselineGateParksEnvironmentFailureWithoutDispatch(t *testing.T) { + tests := []struct { + name string + gate func(t *testing.T, dir string) [][]string + wantCause string + }{ + { + name: "no docker daemon for the test harness", + gate: func(t *testing.T, dir string) [][]string { + argv := gateScript(t, dir, + "echo 'pgtest: start postgres:18 (is Docker running?): rootless Docker not found, failed to create Docker provider'\n"+ + "echo 'FAIL\tinternal/webui'\n"+ + "exit 1") + return [][]string{argv} + }, + wantCause: "Docker", + }, + { + name: "the gate command is not installed", + gate: func(t *testing.T, _ string) [][]string { + return [][]string{{"no-such-gate-binary-archie"}} + }, + wantCause: "could not be started", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + dispatched := false + runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) { + dispatched = true + // Report failure so the red-stub run reaches the dispatch + // assertion instead of a nil-tree commit after a passed result. + return agentexec.Result{Version: agentexec.ProtocolVersion, Status: "failed"}, nil + }) + tc := &TaskContext{ + Task: &Task{ID: 1, Owner: "o", Repo: "r"}, + Repo: config.Repo{Owner: "o", Name: "r", Gate: tt.gate(t, dir)}, + Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}}, + Agent: runner, Trees: &fakeTrees{}, + Dir: dir, + Log: slog.New(slog.DiscardHandler), + } + + err := StageBaselineGate().Run(context.Background(), tc) + if err == nil { + t.Fatal("environment-caused gate failure returned nil, want a park error") + } + if dispatched { + t.Fatal("an environment-caused gate failure dispatched the baseline-fix builder") + } + if !strings.Contains(err.Error(), tt.wantCause) { + t.Fatalf("park error does not name the cause %q: %v", tt.wantCause, err) + } + }) + } +} + +// TestStageBaselineGateDispatchesBuilderForCodeFailure is acceptance (b): a +// genuine pre-existing code failure still reaches the repair agent. +func TestStageBaselineGateDispatchesBuilderForCodeFailure(t *testing.T) { + dir := t.TempDir() + argv := gateScript(t, dir, "echo '--- FAIL: TestBroken (0.00s)'; echo 'FAIL\tpkg/x'; exit 1") + + dispatched := false + runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) { + dispatched = true + return agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed}, nil + }) + tc := &TaskContext{ + Task: &Task{ID: 1, Owner: "o", Repo: "r"}, + Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}}, + Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}}, + Agent: runner, Trees: &fakeTrees{}, + Dir: dir, + Log: slog.New(slog.DiscardHandler), + } + + if err := StageBaselineGate().Run(context.Background(), tc); err != nil { + t.Fatalf("StageBaselineGate.Run() = %v, want nil", err) + } + if !dispatched { + t.Fatal("a code failure did not dispatch the baseline-fix builder") + } +} + +// TestStageBaselineGateProviderFailureStillReportsGateOutput is acceptance +// (c): a provider error during the fix run must not replace the gate cause -- +// the park reason carries the provider failure and the gate output together. +func TestStageBaselineGateProviderFailureStillReportsGateOutput(t *testing.T) { + dir := t.TempDir() + argv := gateScript(t, dir, + "echo '--- FAIL: TestBroken (0.00s)'\n"+ + "echo ' broken_test.go:5: pre-existing failure'\n"+ + "exit 1") + + runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) { + return agentexec.Result{}, errors.New("provider: 402 Payment Required") + }) + tc := &TaskContext{ + Task: &Task{ID: 1, Owner: "o", Repo: "r"}, + Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}}, + Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}}, + Agent: runner, + Dir: dir, + Log: slog.New(slog.DiscardHandler), + } + + err := StageBaselineGate().Run(context.Background(), tc) + if err == nil { + t.Fatal("provider failure during the fix run returned nil, want a park error") + } + if !strings.Contains(err.Error(), "402") { + t.Fatalf("park error dropped the provider failure: %v", err) + } + if !strings.Contains(err.Error(), "TestBroken") { + t.Fatalf("provider failure masked the gate output: %v", err) + } +} diff --git a/internal/domain/workflow/implement.go b/internal/domain/workflow/implement.go index afd7d550b..7fd717fc9 100644 --- a/internal/domain/workflow/implement.go +++ b/internal/domain/workflow/implement.go @@ -52,75 +52,91 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error { if err == nil { continue } + // A failure the environment itself caused -- the command could not + // run, or its output names a host resource the gate needs and does + // not have -- is not a pre-existing code defect: no repair agent in + // the worktree can fix it. Park with the cause and the output rather + // than spend a run budget on a builder that cannot succeed + // (archie-core-rwy6). + if cause := gateEnvironmentCause(string(out), err); cause != "" { + return baselineEnvironmentError(argv, string(out), cause) + } tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure", "cmd", strings.Join(argv, " "), "err", clipTail(string(out), baselineWarnLogBytes)) + if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil { + return fixErr + } + } + return nil +} - // Run builder to fix the failure via TDD. - mission := fmt.Sprintf( - "The baseline gate check `%s` failed on the repository %s at the base commit "+ - "(before any feature work). This is a pre-existing issue -- not caused by your work. "+ - "Fix it using TDD:\n\n"+ - "1. Write a test that proves the failure exists\n"+ - "2. Run the test -- it should FAIL (confirming the bug)\n"+ - "3. Fix the root cause\n"+ - "4. Run the test -- it should PASS\n"+ - "5. Repeat until the gate `%s` passes for all packages\n\n"+ - "Gate output:\n%s\n\n"+ - "When the gate passes, call finish with status \"passed\".", - strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "), - clip(extractFailingGateOutput(string(out)), baselineMissionBytes), - ) +// runBaselineFix dispatches the builder to repair one pre-existing gate +// failure via TDD and commits the result. It is reached only for a code +// failure: an environment failure is parked by the caller before it runs. +func runBaselineFix(ctx context.Context, tc *TaskContext, argv []string, out []byte) error { + mission := fmt.Sprintf( + "The baseline gate check `%s` failed on the repository %s at the base commit "+ + "(before any feature work). This is a pre-existing issue -- not caused by your work. "+ + "Fix it using TDD:\n\n"+ + "1. Write a test that proves the failure exists\n"+ + "2. Run the test -- it should FAIL (confirming the bug)\n"+ + "3. Fix the root cause\n"+ + "4. Run the test -- it should PASS\n"+ + "5. Repeat until the gate `%s` passes for all packages\n\n"+ + "Gate output:\n%s\n\n"+ + "When the gate passes, call finish with status \"passed\".", + strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "), + clip(extractFailingGateOutput(string(out)), baselineMissionBytes), + ) - modelRef := tc.Cfg.Models["builder"] - req := agentexec.Request{ - Version: agentexec.ProtocolVersion, - TaskID: tc.Task.ID, - Attempt: tc.Task.Attempt, - Stage: "baseline-fix", - Workflow: tc.Task.Workflow, - Model: modelRef, - ContextWindow: modelContextBudget(tc.Cfg, modelRef), - Mission: mission, - Budget: agentexec.Budget{ - MaxSteps: tc.Cfg.Budgets.MaxSteps, - WallClock: tc.Cfg.Budgets.WallClock.Std(), - }, - Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets), - Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)}, - } + modelRef := tc.Cfg.Models["builder"] + req := agentexec.Request{ + Version: agentexec.ProtocolVersion, + TaskID: tc.Task.ID, + Attempt: tc.Task.Attempt, + Stage: "baseline-fix", + Workflow: tc.Task.Workflow, + Model: modelRef, + ContextWindow: modelContextBudget(tc.Cfg, modelRef), + Mission: mission, + Budget: agentexec.Budget{ + MaxSteps: tc.Cfg.Budgets.MaxSteps, + WallClock: tc.Cfg.Budgets.WallClock.Std(), + }, + Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets), + Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)}, + } - res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) { - return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix")) - }) - if agentErr != nil && res.Version == 0 { - return fmt.Errorf("baseline-fix agent run: %w", agentErr) - } - tc.Task.TokensUsed += res.TokensUsed - tc.Task.Iterations += res.Iterations - accumulateUsage(&tc.RunUsage, res.Usage) - if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil { - return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr) - } - if agentErr != nil { - return fmt.Errorf("baseline-fix agent run: %w", agentErr) - } - if res.Status != agentexec.StatusPassed { - return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s", - strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes)) - } + res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) { + return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix")) + }) + if agentErr != nil && res.Version == 0 { + return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr)) + } + tc.Task.TokensUsed += res.TokensUsed + tc.Task.Iterations += res.Iterations + accumulateUsage(&tc.RunUsage, res.Usage) + if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil { + return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr) + } + if agentErr != nil { + return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr)) + } + if res.Status != agentexec.StatusPassed { + return baselineFixFailedError(argv, string(out), "builder status "+res.Status) + } - // Commit the baseline fix. - changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir, - fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " "))) - if commitErr != nil { - tc.Log.Warn("baseline fix commit failed", "err", commitErr) - } - if changed { - tc.BuildSummary = res.Summary - tc.BaselineFixed = true - tc.captureChanges(ctx, capturedAfterBaselineFix) - } + // Commit the baseline fix. + changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir, + fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " "))) + if commitErr != nil { + tc.Log.Warn("baseline fix commit failed", "err", commitErr) + } + if changed { + tc.BuildSummary = res.Summary + tc.BaselineFixed = true + tc.captureChanges(ctx, capturedAfterBaselineFix) } return nil }