fix(workflow): park environment gate failures without dispatching a builder
The baseline stage treated every red gate as a pre-existing code defect and dispatched a TDD repair agent. A failure the environment caused -- no Docker daemon for the test harness, or a gate command that is not installed -- cannot be fixed from the worktree, so the run spent a full agent budget and then parked. A provider error during that repair run (observed: HTTP 402) replaced the gate cause in the park reason entirely. Classify at the producer: gateEnvironmentCause names a failure whose command could not start (*exec.Error) or whose output names unavailable host infrastructure (the Docker shapes from archie-core-71mq). That failure parks with the cause and the gate output and starts no builder. A real code failure still dispatches exactly one repair run, and every post-dispatch failure now carries the gate output alongside the agent/provider error instead of masking it. Refs archie-core-rwy6.
This commit is contained in:
1 parent
95e88245da
commit
f1188562ee
3 files changed
+375
-62
No files matched your search
@@ -0,0 +1,87 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// gateEnvironmentCause reports why a failed gate command is an environment
|
||||
// failure the baseline repair agent cannot fix, or "" when the failure is an
|
||||
// ordinary code failure the agent can act on.
|
||||
//
|
||||
// Only two signals are treated as reliable enough to act on:
|
||||
//
|
||||
// - the command could not be started at all. os/exec reports a missing or
|
||||
// non-executable binary as *exec.Error before the gate runs a line of its
|
||||
// own, and no source change in the repository can make a tool that is not
|
||||
// installed appear.
|
||||
// - the combined output names host infrastructure the gate needs and does
|
||||
// not have, such as a Docker daemon the test harness spawns containers on.
|
||||
// These are the phrasings the tools themselves emit when their dependency
|
||||
// is absent (archie-core-71mq), not an inference drawn from the exit
|
||||
// status.
|
||||
//
|
||||
// The second signal is deliberately a closed, literal set: a marker is added
|
||||
// with the incident that produced it, never by inference. A false negative
|
||||
// only spends the repair budget a code failure would have spent anyway, but a
|
||||
// false positive would park a task the builder could have fixed.
|
||||
func gateEnvironmentCause(out string, runErr error) string {
|
||||
if execErr, ok := errors.AsType[*exec.Error](runErr); ok {
|
||||
return "the gate command could not be started (" + execErr.Error() + ")"
|
||||
}
|
||||
lower := strings.ToLower(out)
|
||||
for _, m := range gateEnvironmentMarkers {
|
||||
if strings.Contains(lower, m.marker) {
|
||||
return m.cause
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// gateEnvironmentMarkers pairs an output fragment a missing host resource
|
||||
// produces with the cause it names. Markers are matched case-insensitively
|
||||
// against the gate's combined output.
|
||||
var gateEnvironmentMarkers = []struct {
|
||||
marker string
|
||||
cause string
|
||||
}{
|
||||
{
|
||||
marker: "is docker running?",
|
||||
cause: "the Docker daemon required by the test harness is unavailable",
|
||||
},
|
||||
{
|
||||
marker: "cannot connect to the docker daemon",
|
||||
cause: "the Docker daemon is unavailable",
|
||||
},
|
||||
{
|
||||
marker: "rootless docker not found",
|
||||
cause: "the Docker daemon is unavailable",
|
||||
},
|
||||
{
|
||||
marker: "failed to create docker provider",
|
||||
cause: "the Docker daemon is unavailable",
|
||||
},
|
||||
{
|
||||
marker: "docker: command not found",
|
||||
cause: "the docker CLI is not installed",
|
||||
},
|
||||
}
|
||||
|
||||
// baselineEnvironmentError is the park reason when a gate failed for an
|
||||
// environment cause: it names the cause and keeps the gate output an operator
|
||||
// needs to confirm it. The stage returns this without dispatching a builder.
|
||||
func baselineEnvironmentError(argv []string, out, cause string) error {
|
||||
return fmt.Errorf("baseline red -- %s cannot run in this environment: %s\n\ngate output:\n%s",
|
||||
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
|
||||
}
|
||||
|
||||
// baselineFixFailedError is the park reason for every failure after the
|
||||
// baseline repair agent was dispatched. A failure that names only the agent --
|
||||
// a provider 402, say -- and drops the gate output leaves the operator with no
|
||||
// way to see why the gate was red in the first place (archie-core-rwy6).
|
||||
func baselineFixFailedError(argv []string, out, cause string) error {
|
||||
return fmt.Errorf("baseline red -- %s fails and the baseline-fix builder did not repair it: %s\n\ngate output:\n%s",
|
||||
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
)
|
||||
|
||||
// gateScript writes an executable shell script running `body` and returns
|
||||
// the argv that invokes it.
|
||||
func gateScript(t *testing.T, dir, body string) []string {
|
||||
t.Helper()
|
||||
path := filepath.Join(dir, "gate.sh")
|
||||
if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return []string{"sh", path}
|
||||
}
|
||||
|
||||
// TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures pins the
|
||||
// classification the baseline stage acts on. It must recognise the shapes a
|
||||
// missing host resource produces (the command could not start at all, or a
|
||||
// tool says its dependency is absent) and must not mistake an ordinary code
|
||||
// failure for one of them -- a false positive parks a task the builder could
|
||||
// have fixed.
|
||||
func TestGateEnvironmentCauseClassifiesOnlyInfrastructureFailures(t *testing.T) {
|
||||
failingExit := errors.New("exit status 1")
|
||||
missingBinary := &exec.Error{Name: "go", Err: exec.ErrNotFound}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
out string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "pgtest cannot reach the docker daemon",
|
||||
out: "pgtest: start postgres:18 (is Docker running?): " +
|
||||
"rootless Docker not found, failed to create Docker provider\nFAIL\tinternal/webui",
|
||||
err: failingExit,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "testcontainers cannot connect to the daemon",
|
||||
out: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?",
|
||||
err: failingExit,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "the gate binary is not installed",
|
||||
err: missingBinary,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "an ordinary test failure is a code failure",
|
||||
out: "--- FAIL: TestBroken (0.00s)\n broken_test.go:5: want 1, got 2\nFAIL\tpkg/x",
|
||||
err: failingExit,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "a compile error is a code failure",
|
||||
out: "# pkg/x\n./x.go:3:2: undefined: missing",
|
||||
err: failingExit,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cause := gateEnvironmentCause(tt.out, tt.err)
|
||||
if got := cause != ""; got != tt.want {
|
||||
t.Fatalf("gateEnvironmentCause() = %q, want environmental=%v", cause, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageBaselineGateParksEnvironmentFailureWithoutDispatch is acceptance
|
||||
// (a): a gate that failed because the environment lacks a resource parks with
|
||||
// that cause and never starts the baseline-fix builder -- nothing in the
|
||||
// worktree can fix a missing Docker daemon, so dispatching one only burns the
|
||||
// run budget.
|
||||
func TestStageBaselineGateParksEnvironmentFailureWithoutDispatch(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
gate func(t *testing.T, dir string) [][]string
|
||||
wantCause string
|
||||
}{
|
||||
{
|
||||
name: "no docker daemon for the test harness",
|
||||
gate: func(t *testing.T, dir string) [][]string {
|
||||
argv := gateScript(t, dir,
|
||||
"echo 'pgtest: start postgres:18 (is Docker running?): rootless Docker not found, failed to create Docker provider'\n"+
|
||||
"echo 'FAIL\tinternal/webui'\n"+
|
||||
"exit 1")
|
||||
return [][]string{argv}
|
||||
},
|
||||
wantCause: "Docker",
|
||||
},
|
||||
{
|
||||
name: "the gate command is not installed",
|
||||
gate: func(t *testing.T, _ string) [][]string {
|
||||
return [][]string{{"no-such-gate-binary-archie"}}
|
||||
},
|
||||
wantCause: "could not be started",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dispatched := false
|
||||
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
dispatched = true
|
||||
// Report failure so the red-stub run reaches the dispatch
|
||||
// assertion instead of a nil-tree commit after a passed result.
|
||||
return agentexec.Result{Version: agentexec.ProtocolVersion, Status: "failed"}, nil
|
||||
})
|
||||
tc := &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
|
||||
Repo: config.Repo{Owner: "o", Name: "r", Gate: tt.gate(t, dir)},
|
||||
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
|
||||
Agent: runner, Trees: &fakeTrees{},
|
||||
Dir: dir,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
|
||||
err := StageBaselineGate().Run(context.Background(), tc)
|
||||
if err == nil {
|
||||
t.Fatal("environment-caused gate failure returned nil, want a park error")
|
||||
}
|
||||
if dispatched {
|
||||
t.Fatal("an environment-caused gate failure dispatched the baseline-fix builder")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tt.wantCause) {
|
||||
t.Fatalf("park error does not name the cause %q: %v", tt.wantCause, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageBaselineGateDispatchesBuilderForCodeFailure is acceptance (b): a
|
||||
// genuine pre-existing code failure still reaches the repair agent.
|
||||
func TestStageBaselineGateDispatchesBuilderForCodeFailure(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
argv := gateScript(t, dir, "echo '--- FAIL: TestBroken (0.00s)'; echo 'FAIL\tpkg/x'; exit 1")
|
||||
|
||||
dispatched := false
|
||||
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
dispatched = true
|
||||
return agentexec.Result{Version: agentexec.ProtocolVersion, Status: agentexec.StatusPassed}, nil
|
||||
})
|
||||
tc := &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
|
||||
Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}},
|
||||
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
|
||||
Agent: runner, Trees: &fakeTrees{},
|
||||
Dir: dir,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
|
||||
if err := StageBaselineGate().Run(context.Background(), tc); err != nil {
|
||||
t.Fatalf("StageBaselineGate.Run() = %v, want nil", err)
|
||||
}
|
||||
if !dispatched {
|
||||
t.Fatal("a code failure did not dispatch the baseline-fix builder")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageBaselineGateProviderFailureStillReportsGateOutput is acceptance
|
||||
// (c): a provider error during the fix run must not replace the gate cause --
|
||||
// the park reason carries the provider failure and the gate output together.
|
||||
func TestStageBaselineGateProviderFailureStillReportsGateOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
argv := gateScript(t, dir,
|
||||
"echo '--- FAIL: TestBroken (0.00s)'\n"+
|
||||
"echo ' broken_test.go:5: pre-existing failure'\n"+
|
||||
"exit 1")
|
||||
|
||||
runner := agentRunnerFunc(func(context.Context, string, agentexec.Request, agentexec.ToolCallReporter) (agentexec.Result, error) {
|
||||
return agentexec.Result{}, errors.New("provider: 402 Payment Required")
|
||||
})
|
||||
tc := &TaskContext{
|
||||
Task: &Task{ID: 1, Owner: "o", Repo: "r"},
|
||||
Repo: config.Repo{Owner: "o", Name: "r", Gate: [][]string{argv}},
|
||||
Cfg: config.Config{Models: map[string]string{"builder": "provider/model"}},
|
||||
Agent: runner,
|
||||
Dir: dir,
|
||||
Log: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
|
||||
err := StageBaselineGate().Run(context.Background(), tc)
|
||||
if err == nil {
|
||||
t.Fatal("provider failure during the fix run returned nil, want a park error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "402") {
|
||||
t.Fatalf("park error dropped the provider failure: %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "TestBroken") {
|
||||
t.Fatalf("provider failure masked the gate output: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -52,75 +52,91 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
// A failure the environment itself caused -- the command could not
|
||||
// run, or its output names a host resource the gate needs and does
|
||||
// not have -- is not a pre-existing code defect: no repair agent in
|
||||
// the worktree can fix it. Park with the cause and the output rather
|
||||
// than spend a run budget on a builder that cannot succeed
|
||||
// (archie-core-rwy6).
|
||||
if cause := gateEnvironmentCause(string(out), err); cause != "" {
|
||||
return baselineEnvironmentError(argv, string(out), cause)
|
||||
}
|
||||
tc.Log.Warn("baseline red -- auto-fixing pre-existing gate failure",
|
||||
"cmd", strings.Join(argv, " "),
|
||||
"err", clipTail(string(out), baselineWarnLogBytes))
|
||||
if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil {
|
||||
return fixErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run builder to fix the failure via TDD.
|
||||
mission := fmt.Sprintf(
|
||||
"The baseline gate check `%s` failed on the repository %s at the base commit "+
|
||||
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
|
||||
"Fix it using TDD:\n\n"+
|
||||
"1. Write a test that proves the failure exists\n"+
|
||||
"2. Run the test -- it should FAIL (confirming the bug)\n"+
|
||||
"3. Fix the root cause\n"+
|
||||
"4. Run the test -- it should PASS\n"+
|
||||
"5. Repeat until the gate `%s` passes for all packages\n\n"+
|
||||
"Gate output:\n%s\n\n"+
|
||||
"When the gate passes, call finish with status \"passed\".",
|
||||
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
|
||||
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
|
||||
)
|
||||
// runBaselineFix dispatches the builder to repair one pre-existing gate
|
||||
// failure via TDD and commits the result. It is reached only for a code
|
||||
// failure: an environment failure is parked by the caller before it runs.
|
||||
func runBaselineFix(ctx context.Context, tc *TaskContext, argv []string, out []byte) error {
|
||||
mission := fmt.Sprintf(
|
||||
"The baseline gate check `%s` failed on the repository %s at the base commit "+
|
||||
"(before any feature work). This is a pre-existing issue -- not caused by your work. "+
|
||||
"Fix it using TDD:\n\n"+
|
||||
"1. Write a test that proves the failure exists\n"+
|
||||
"2. Run the test -- it should FAIL (confirming the bug)\n"+
|
||||
"3. Fix the root cause\n"+
|
||||
"4. Run the test -- it should PASS\n"+
|
||||
"5. Repeat until the gate `%s` passes for all packages\n\n"+
|
||||
"Gate output:\n%s\n\n"+
|
||||
"When the gate passes, call finish with status \"passed\".",
|
||||
strings.Join(argv, " "), tc.Repo.FullName(), strings.Join(argv, " "),
|
||||
clip(extractFailingGateOutput(string(out)), baselineMissionBytes),
|
||||
)
|
||||
|
||||
modelRef := tc.Cfg.Models["builder"]
|
||||
req := agentexec.Request{
|
||||
Version: agentexec.ProtocolVersion,
|
||||
TaskID: tc.Task.ID,
|
||||
Attempt: tc.Task.Attempt,
|
||||
Stage: "baseline-fix",
|
||||
Workflow: tc.Task.Workflow,
|
||||
Model: modelRef,
|
||||
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
|
||||
Mission: mission,
|
||||
Budget: agentexec.Budget{
|
||||
MaxSteps: tc.Cfg.Budgets.MaxSteps,
|
||||
WallClock: tc.Cfg.Budgets.WallClock.Std(),
|
||||
},
|
||||
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
|
||||
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
|
||||
}
|
||||
modelRef := tc.Cfg.Models["builder"]
|
||||
req := agentexec.Request{
|
||||
Version: agentexec.ProtocolVersion,
|
||||
TaskID: tc.Task.ID,
|
||||
Attempt: tc.Task.Attempt,
|
||||
Stage: "baseline-fix",
|
||||
Workflow: tc.Task.Workflow,
|
||||
Model: modelRef,
|
||||
ContextWindow: modelContextBudget(tc.Cfg, modelRef),
|
||||
Mission: mission,
|
||||
Budget: agentexec.Budget{
|
||||
MaxSteps: tc.Cfg.Budgets.MaxSteps,
|
||||
WallClock: tc.Cfg.Budgets.WallClock.Std(),
|
||||
},
|
||||
Gate: GateFromRepo(tc.Repo, tc.Cfg.Budgets),
|
||||
Protection: agentexec.Protection{Suffixes: append([]string(nil), tc.Repo.Protect...)},
|
||||
}
|
||||
|
||||
res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) {
|
||||
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
|
||||
})
|
||||
if agentErr != nil && res.Version == 0 {
|
||||
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
|
||||
}
|
||||
tc.Task.TokensUsed += res.TokensUsed
|
||||
tc.Task.Iterations += res.Iterations
|
||||
accumulateUsage(&tc.RunUsage, res.Usage)
|
||||
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
|
||||
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
|
||||
}
|
||||
if agentErr != nil {
|
||||
return fmt.Errorf("baseline-fix agent run: %w", agentErr)
|
||||
}
|
||||
if res.Status != agentexec.StatusPassed {
|
||||
return fmt.Errorf("baseline red -- %s fails and builder could not auto-fix (status: %s)\n\ngate output:\n%s",
|
||||
strings.Join(argv, " "), res.Status, clipTail(string(out), baselineParkOutputBytes))
|
||||
}
|
||||
res, agentErr := tc.RunAgentChild(ctx, "baseline-fix", func() (agentexec.Result, error) {
|
||||
return tc.Agent.Run(ctx, tc.Dir, req, tc.toolCallReporter("baseline-fix"))
|
||||
})
|
||||
if agentErr != nil && res.Version == 0 {
|
||||
return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr))
|
||||
}
|
||||
tc.Task.TokensUsed += res.TokensUsed
|
||||
tc.Task.Iterations += res.Iterations
|
||||
accumulateUsage(&tc.RunUsage, res.Usage)
|
||||
if emitErr := tc.EmitDurable(ctx, events.KindAgentFinish, "baseline-fix", res.Summary, agentFinishData(res, modelRef)); emitErr != nil {
|
||||
return fmt.Errorf("persist baseline-fix agent finish: %w", emitErr)
|
||||
}
|
||||
if agentErr != nil {
|
||||
return baselineFixFailedError(argv, string(out), fmt.Sprintf("the builder could not run: %v", agentErr))
|
||||
}
|
||||
if res.Status != agentexec.StatusPassed {
|
||||
return baselineFixFailedError(argv, string(out), "builder status "+res.Status)
|
||||
}
|
||||
|
||||
// Commit the baseline fix.
|
||||
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
|
||||
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
|
||||
if commitErr != nil {
|
||||
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
|
||||
}
|
||||
if changed {
|
||||
tc.BuildSummary = res.Summary
|
||||
tc.BaselineFixed = true
|
||||
tc.captureChanges(ctx, capturedAfterBaselineFix)
|
||||
}
|
||||
// Commit the baseline fix.
|
||||
changed, commitErr := tc.Trees.CommitAll(ctx, tc.Dir,
|
||||
fmt.Sprintf("fix: baseline gate repair (%s)", strings.Join(argv, " ")))
|
||||
if commitErr != nil {
|
||||
tc.Log.Warn("baseline fix commit failed", "err", commitErr)
|
||||
}
|
||||
if changed {
|
||||
tc.BuildSummary = res.Summary
|
||||
tc.BaselineFixed = true
|
||||
tc.captureChanges(ctx, capturedAfterBaselineFix)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user