fix(workflow): park environment gate failures without dispatching a builder

The baseline stage treated every red gate as a pre-existing code defect and
dispatched a TDD repair agent. A failure the environment caused -- no Docker
daemon for the test harness, or a gate command that is not installed -- cannot
be fixed from the worktree, so the run spent a full agent budget and then
parked. A provider error during that repair run (observed: HTTP 402) replaced
the gate cause in the park reason entirely.

Classify at the producer: gateEnvironmentCause names a failure whose command
could not start (*exec.Error) or whose output names unavailable host
infrastructure (the Docker shapes from archie-core-71mq). That failure parks
with the cause and the gate output and starts no builder. A real code failure
still dispatches exactly one repair run, and every post-dispatch failure now
carries the gate output alongside the agent/provider error instead of masking
it.

Refs archie-core-rwy6.
This commit is contained in:
sam committed 2026-10-02 09:57:15 +10:00
1 parent 95e88245da
commit f1188562ee
3 files changed
+375 -62

No files matched your search

@@ -0,0 +1,87 @@
package workflow
import (
"errors"
"fmt"
"os/exec"
"strings"
)
// gateEnvironmentCause reports why a failed gate command is an environment
// failure the baseline repair agent cannot fix, or "" when the failure is an
// ordinary code failure the agent can act on.
//
// Only two signals are treated as reliable enough to act on:
//
// - the command could not be started at all. os/exec reports a missing or
// non-executable binary as *exec.Error before the gate runs a line of its
// own, and no source change in the repository can make a tool that is not
// installed appear.
// - the combined output names host infrastructure the gate needs and does
// not have, such as a Docker daemon the test harness spawns containers on.
// These are the phrasings the tools themselves emit when their dependency
// is absent (archie-core-71mq), not an inference drawn from the exit
// status.
//
// The second signal is deliberately a closed, literal set: a marker is added
// with the incident that produced it, never by inference. A false negative
// only spends the repair budget a code failure would have spent anyway, but a
// false positive would park a task the builder could have fixed.
func gateEnvironmentCause(out string, runErr error) string {
if execErr, ok := errors.AsType[*exec.Error](runErr); ok {
return "the gate command could not be started (" + execErr.Error() + ")"
}
lower := strings.ToLower(out)
for _, m := range gateEnvironmentMarkers {
if strings.Contains(lower, m.marker) {
return m.cause
}
}
return ""
}
// gateEnvironmentMarkers pairs an output fragment a missing host resource
// produces with the cause it names. Markers are matched case-insensitively
// against the gate's combined output.
var gateEnvironmentMarkers = []struct {
marker string
cause string
}{
{
marker: "is docker running?",
cause: "the Docker daemon required by the test harness is unavailable",
},
{
marker: "cannot connect to the docker daemon",
cause: "the Docker daemon is unavailable",
},
{
marker: "rootless docker not found",
cause: "the Docker daemon is unavailable",
},
{
marker: "failed to create docker provider",
cause: "the Docker daemon is unavailable",
},
{
marker: "docker: command not found",
cause: "the docker CLI is not installed",
},
}
// baselineEnvironmentError is the park reason when a gate failed for an
// environment cause: it names the cause and keeps the gate output an operator
// needs to confirm it. The stage returns this without dispatching a builder.
func baselineEnvironmentError(argv []string, out, cause string) error {
return fmt.Errorf("baseline red -- %s cannot run in this environment: %s\n\ngate output:\n%s",
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
}
// baselineFixFailedError is the park reason for every failure after the
// baseline repair agent was dispatched. A failure that names only the agent --
// a provider 402, say -- and drops the gate output leaves the operator with no
// way to see why the gate was red in the first place (archie-core-rwy6).
func baselineFixFailedError(argv []string, out, cause string) error {
return fmt.Errorf("baseline red -- %s fails and the baseline-fix builder did not repair it: %s\n\ngate output:\n%s",
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
}
@@ -0,0 +1,210 @@
package workflow
import (
"context"
"errors"
"log/slog"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"github.com/samcharles93/archie-core/internal/agentexec"
"github.com/samcharles93/archie-core/internal/config"
)
// gateScript writes an executable shell script running `body` and returns
// the argv that invokes it.
func gateScript(t *testing.T, dir, body string) []string {
t.Helper()
path := filepath.Join(dir, "gate.sh")
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil {
t.Fatal(err)
}
return []string{"sh", path}
}
// TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures pins the
// classification the baseline stage acts on. It must recognise the shapes a
// missing host resource produces (the command could not start at all, or a
// tool says its dependency is absent) and must not mistake an ordinary code
// failure for one of them -- a false positive parks a task the builder could
// have fixed.
func TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures(t *testing.T) {
failingExit := errors.New("exit status 1")
missingBinary := &exec.Error{Name: "go", Err: exec.ErrNotFound}
tests := []struct {
name string
out string
err error
want bool
}{
{
name: "pgtest cannot reach the docker daemon",
out: "pgtest: start postgres:18 (is Docker running?): " +
"rootless Docker not found, failed to create Docker provider\nFAIL\tinternal/webui",
err: failingExit,
want: true,
},
{
name: "testcontainers cannot connect to the daemon",
out: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?",
err: failingExit,
want: true,
},
{
name: "the gate binary is not installed",
err: missingBinary,
want: true,
},
{
name: "an ordinary test failure is a code failure",
out: "--- FAIL: TestBroken (0.00s)\n broken_test.go:5: want 1, got 2\nFAIL\tpkg/x",
err: failingExit,
want: false,
},
{
name: "a compile error is a code failure",
out: "# pkg/x\n./x.go:3:2: undefined: missing",
err: failingExit,
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cause := gateEnvironmentCause(tt.out, tt.err)
if got := cause != ""; got != tt.want {
t.Fatalf("gateEnvironmentCause() = %q, want environmental=%v", cause, tt.want)
}
})
}
}
// TestStageBaselineGateParksEnvironmentFailureWithoutDispatch is acceptance
// (a): a gate that failed because the environment lacks a resource parks with
// that cause and never starts the baseline-fix builder -- nothing in the
// worktree can fix a missing Docker daemon, so dispatching one only burns the
// run budget.
func TestStageBaselineGateParksEnvironmentFailureWithoutDispatch(t *testing.T) {
tests := []struct {
name string
gate func(t *testing.T, dir string) [][]string
wantCause string
}{
{
name: "no docker daemon for the test harness",
gate: func(t *testing.T, dir string) [][]string {
argv := gateScript(t, dir,
"echo 'pgtest: start postgres:18 (is Docker running?): rootless Docker not found, failed to create Docker provider'\n"+
"echo 'FAIL\tinternal/webui'\n"+
"exit 1")
return [][]string{argv}
},
wantCause: "Docker",
},
{
name: "the gate command is not installed",
gate: func(t *testing.T, _ string) [][]string {
return [][]string{{"no-such-gate-binary-archie"}}
},
wantCause: "could not be started",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
dispatched := false
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
dispatched = true
// Report failure so the red-stub run reaches the dispatch
// assertion instead of a nil-tree commit after a passed result.
return agentexec.Result{Version: agentexec.ProtocolVersion, Status: "failed"}, nil
})
tc := &TaskContext{
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
Repo: config.Repo{Owner: "o", Name: "r", Gate: tt.gate(t, dir)},
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
Agent: runner, Trees: &fakeTrees{},
Dir: dir,
Log: slog.New(slog.DiscardHandler),
}
err := StageBaselineGate().Run(context.Background(), tc)
if err == nil {
t.Fatal("environment-caused gate failure returned nil, want a park error")
}
if dispatched {
t.Fatal("an environment-caused gate failure dispatched the baseline-fix builder")
}
if !strings.Contains(err.Error(), tt.wantCause) {
t.Fatalf("park error does not name the cause %q: %v", tt.wantCause, err)
}
})
}
}
// TestStageBaselineGateDispatchesBuilderForCodeFailure is acceptance (b): a
// genuine pre-existing code failure still reaches the repair agent.
func TestStageBaselineGateDispatchesBuilderForCodeFailure(t *testing.T) {
dir := t.TempDir()
argv := gateScript(t, dir, "echo '--- FAIL: TestBroken (0.00s)'; echo 'FAIL\tpkg/x'; exit 1")
dispatched := false
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
dispatched = true
return agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed}, nil
})
tc := &TaskContext{
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}},
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
Agent: runner, Trees: &fakeTrees{},
Dir: dir,
Log: slog.New(slog.DiscardHandler),
}
if err := StageBaselineGate().Run(context.Background(), tc); err != nil {
t.Fatalf("StageBaselineGate.Run() = %v, want nil", err)
}
if !dispatched {
t.Fatal("a code failure did not dispatch the baseline-fix builder")
}
}
// TestStageBaselineGateProviderFailureStillReportsGateOutput is acceptance
// (c): a provider error during the fix run must not replace the gate cause --
// the park reason carries the provider failure and the gate output together.
func TestStageBaselineGateProviderFailureStillReportsGateOutput(t *testing.T) {
dir := t.TempDir()
argv := gateScript(t, dir,
"echo '--- FAIL: TestBroken (0.00s)'\n"+
"echo ' broken_test.go:5: pre-existing failure'\n"+
"exit 1")
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
return agentexec.Result{}, errors.New("provider: 402 Payment Required")
})
tc := &TaskContext{
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}},
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
Agent: runner,
Dir: dir,
Log: slog.New(slog.DiscardHandler),
}
err := StageBaselineGate().Run(context.Background(), tc)
if err == nil {
t.Fatal("provider failure during the fix run returned nil, want a park error")
}
if !strings.Contains(err.Error(), "402") {
t.Fatalf("park error dropped the provider failure: %v", err)
}
if !strings.Contains(err.Error(), "TestBroken") {
t.Fatalf("provider failure masked the gate output: %v", err)
}
}
+78 -62
View File
@@ -52,75 +52,91 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
if err == nil {
continue
}
// A failure the environment itself caused -- the command could not
// run, or its output names a host resource the gate needs and does
// not have -- is not a pre-existing code defect: no repair agent in
// the worktree can fix it. Park with the cause and the output rather
// than spend a run budget on a builder that cannot succeed
// (archie-core-rwy6).
if cause := gateEnvironmentCause(string(out), err); cause != "" {
return baselineEnvironmentError(argv, string(out), cause)
}
tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure",
"cmd", strings.Join(argv, " "),
"err", clipTail(string(out), baselineWarnLogBytes))
if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil {
return fixErr
}
}
return nil
}
// Run builder to fix the failure via TDD.
mission := fmt.Sprintf(
"The baseline gate check `%s` failed on the repository %s at the base commit "+
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
"Fix it using TDD:\n\n"+
"1. Write a test that proves the failure exists\n"+
"2. Run the test -- it should FAIL (confirming the bug)\n"+
"3. Fix the root cause\n"+
"4. Run the test -- it should PASS\n"+
"5. Repeat until the gate `%s` passes for all packages\n\n"+
"Gate output:\n%s\n\n"+
"When the gate passes, call finish with status \"passed\".",
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
)
// runBaselineFix dispatches the builder to repair one pre-existing gate
// failure via TDD and commits the result. It is reached only for a code
// failure: an environment failure is parked by the caller before it runs.
func runBaselineFix(ctx context.Context, tc *TaskContext, argv []string, out []byte) error {
mission := fmt.Sprintf(
"The baseline gate check `%s` failed on the repository %s at the base commit "+
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
"Fix it using TDD:\n\n"+
"1. Write a test that proves the failure exists\n"+
"2. Run the test -- it should FAIL (confirming the bug)\n"+
"3. Fix the root cause\n"+
"4. Run the test -- it should PASS\n"+
"5. Repeat until the gate `%s` passes for all packages\n\n"+
"Gate output:\n%s\n\n"+
"When the gate passes, call finish with status \"passed\".",
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
)
modelRef := tc.Cfg.Models["builder"]
req := agentexec.Request{
Version: agentexec.ProtocolVersion,
TaskID: tc.Task.ID,
Attempt: tc.Task.Attempt,
Stage: "baseline-fix",
Workflow: tc.Task.Workflow,
Model: modelRef,
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
Mission: mission,
Budget: agentexec.Budget{
MaxSteps: tc.Cfg.Budgets.MaxSteps,
WallClock: tc.Cfg.Budgets.WallClock.Std(),
},
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
}
modelRef := tc.Cfg.Models["builder"]
req := agentexec.Request{
Version: agentexec.ProtocolVersion,
TaskID: tc.Task.ID,
Attempt: tc.Task.Attempt,
Stage: "baseline-fix",
Workflow: tc.Task.Workflow,
Model: modelRef,
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
Mission: mission,
Budget: agentexec.Budget{
MaxSteps: tc.Cfg.Budgets.MaxSteps,
WallClock: tc.Cfg.Budgets.WallClock.Std(),
},
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
}
res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) {
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
})
if agentErr != nil && res.Version == 0 {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
tc.Task.TokensUsed += res.TokensUsed
tc.Task.Iterations += res.Iterations
accumulateUsage(&tc.RunUsage, res.Usage)
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
}
if agentErr != nil {
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
}
if res.Status != agentexec.StatusPassed {
return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s",
strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes))
}
res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) {
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
})
if agentErr != nil && res.Version == 0 {
return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr))
}
tc.Task.TokensUsed += res.TokensUsed
tc.Task.Iterations += res.Iterations
accumulateUsage(&tc.RunUsage, res.Usage)
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
}
if agentErr != nil {
return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr))
}
if res.Status != agentexec.StatusPassed {
return baselineFixFailedError(argv, string(out), "builder status "+res.Status)
}
// Commit the baseline fix.
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
if commitErr != nil {
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
}
if changed {
tc.BuildSummary = res.Summary
tc.BaselineFixed = true
tc.captureChanges(ctx, capturedAfterBaselineFix)
}
// Commit the baseline fix.
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
if commitErr != nil {
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
}
if changed {
tc.BuildSummary = res.Summary
tc.BaselineFixed = true
tc.captureChanges(ctx, capturedAfterBaselineFix)
}
return nil
}