fix(workflow): one baseline fix per repository revision, drop download noise from gate output (#1806)
deploy / plan (push) Successful in 14s
deploy / archied-image (push) Canceled after 0s
deploy / agent-image (push) Canceled after 0s
deploy / dist-zip (push) Canceled after 0s
deploy / gate (push) Canceled after 1m2s

This commit is contained in:
sam committed 2026-11-08 17:23:21 +11:00
1 parent 860b2c85e0
commit d11a0cd245
12 files changed
+386 -13

No files matched your search

@@ -33,6 +33,7 @@ import (
// worktreerpc; local git operations run on the bind-mounted worktree.
type remoteTrees interface {
Push(ctx context.Context) error
workflow.BaselineLedger
}
type hybridTrees struct {
@@ -263,6 +264,9 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen
Log: log,
Guardrails: dependencies.guardrails,
}
if dependencies.trees != nil {
tc.Baseline = dependencies.trees
}
tc.Instructions = req.Instructions
configureTaskContext(tc, workDir)
+1 -1
View File
@@ -362,7 +362,7 @@ func registerTaskRPCServers(nc *natsio.Conn, resolve func(context.Context, *work
resolved, err := resolve(ctx, t, config.OperationPR)
return resolved.Forge, err
}}
trees := &worktreerpc.Server{Runs: runs, Log: log, Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) {
trees := &worktreerpc.Server{Runs: runs, Log: log, Baselines: worktreerpc.NewBaselineClaims(), Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) {
resolved, err := resolve(ctx, t, config.OperationPush)
return resolved.Trees, err
}}
+1
View File
@@ -35,6 +35,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s
agentexec.SubjectForEvents(taskID),
spanrelay.Subject(taskID),
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectBaseline),
messagerpc.SubjectFor(task.Identity),
}, forgerpc.Subjects(task.Identity)...)
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
@@ -0,0 +1,95 @@
package workflow
import (
"context"
"fmt"
"os/exec"
"regexp"
"strings"
"time"
)
// BaselineClaim answers a task's request to repair a red baseline.
type BaselineClaim struct {
// Lead is true when the caller owns the repair and must run it.
Lead bool
// Task is the task that owns the repair when Lead is false.
Task int64
// Done is true once that task has finished; Failure is its failure, empty
// when it repaired the baseline on its own branch.
Done bool
Failure string
}
// BaselineLedger makes one task repair a red baseline per repository revision
// and command. Every task runs in its own container, so the ledger lives with
// the daemon.
type BaselineLedger interface {
ClaimBaselineFix(ctx context.Context, key string) (BaselineClaim, error)
FinishBaselineFix(ctx context.Context, key, failure string) error
}
const defaultBaselinePoll = 15 * time.Second
// repairBaselineOnce runs the repair when this task leads it, or reports the
// verdict of the task that does. Without a ledger every task repairs its own.
func repairBaselineOnce(ctx context.Context, tc *TaskContext, argv []string, out []byte) error {
if tc.Baseline == nil {
return runBaselineFix(ctx, tc, argv, out)
}
revision, err := exec.CommandContext(ctx, "git", "-C", tc.Dir, "rev-parse", "HEAD").Output()
if err != nil {
return fmt.Errorf("resolve baseline revision: %w", err)
}
key := strings.TrimSpace(string(revision)) + " " + strings.Join(argv, " ")
poll := tc.baselinePoll
if poll <= 0 {
poll = defaultBaselinePoll
}
for {
claim, err := tc.Baseline.ClaimBaselineFix(ctx, key)
if err != nil {
return fmt.Errorf("claim baseline fix: %w", err)
}
switch {
case claim.Lead:
fixErr := runBaselineFix(ctx, tc, argv, out)
failure := ""
if fixErr != nil {
failure = clip(fixErr.Error(), baselineParkOutputBytes)
}
if err := tc.Baseline.FinishBaselineFix(context.WithoutCancel(ctx), key, failure); err != nil {
tc.Log.WarnContext(ctx, "release baseline fix claim", "err", err)
}
return fixErr
case !claim.Done:
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(poll):
}
default:
return baselineSharedError(argv, string(out), claim)
}
}
}
// gateProgressLine matches the package-download chatter `go` prints before the
// real failure.
var gateProgressLine = regexp.MustCompile(`^go: (downloading|finding|extracting) `)
// trimGateProgress drops download progress from gate output so the failing
// command's own lines lead.
func trimGateProgress(out string) string {
if !strings.Contains(out, "go: ") {
return out
}
lines := strings.Split(out, "\n")
kept := lines[:0]
for _, line := range lines {
if !gateProgressLine.MatchString(line) {
kept = append(kept, line)
}
}
return strings.Join(kept, "\n")
}
@@ -0,0 +1,110 @@
package workflow
import (
"context"
"log/slog"
"os/exec"
"strings"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/agentrun"
"github.com/samcharles93/archie-core/internal/domain/workflow/task"
)
type countingRunner struct{ runs int }
func (r *countingRunner) Run(context.Context, string, agentrun.Request, agentrun.Reporter) (agentrun.Result, error) {
r.runs++
return agentrun.Result{Version: agentrun.ProtocolVersion, Status: "failed"}, nil
}
// scriptedLedger answers claims from a fixed list, then repeats the last one.
type scriptedLedger struct {
claims []BaselineClaim
finished []string
}
func (l *scriptedLedger) ClaimBaselineFix(context.Context, string) (BaselineClaim, error) {
claim := l.claims[0]
if len(l.claims) > 1 {
l.claims = l.claims[1:]
}
return claim, nil
}
func (l *scriptedLedger) FinishBaselineFix(_ context.Context, _, failure string) error {
l.finished = append(l.finished, failure)
return nil
}
func TestRedBaselineIsRepairedByOneTask(t *testing.T) {
tests := []struct {
name string
claims []BaselineClaim
wantRuns int
wantFinished int
wantInErr string
}{
{
name: "the task that leads repairs it",
claims: []BaselineClaim{{Lead: true}},
wantRuns: 1,
wantFinished: 1,
wantInErr: "did not repair it",
},
{
name: "a task waits on the lead, then reports its failure",
claims: []BaselineClaim{{Task: 7}, {Task: 7}, {Task: 7, Done: true, Failure: "builder status failed"}},
wantInErr: "task 7 could not repair it: builder status failed",
},
{
name: "a task reuses a repair made on another branch",
claims: []BaselineClaim{{Task: 7, Done: true}},
wantInErr: "task 7 repaired it on its own branch",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dir := t.TempDir()
for _, args := range [][]string{{"init", "-q"}, {"-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "base"}} {
if out, err := exec.CommandContext(t.Context(), "git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
}
agent := &countingRunner{}
ledger := &scriptedLedger{claims: tt.claims}
tc := &TaskContext{
Task: &task.Task{ID: 1, Attempt: 1, Source: task.SourceTest},
Repo: config.Repo{Owner: "acme", Name: "widgets"},
Cfg: config.Config{Models: map[string]string{"default": "m"}},
Dir: dir,
Log: slog.Default(),
Agent: agent,
Baseline: ledger,
baselinePoll: time.Millisecond,
}
err := repairBaselineOnce(context.Background(), tc, []string{"task", "check"}, []byte("FAIL"))
if err == nil || !strings.Contains(err.Error(), tt.wantInErr) {
t.Fatalf("err = %v, want it to contain %q", err, tt.wantInErr)
}
if agent.runs != tt.wantRuns {
t.Errorf("fix agents run = %d, want %d", agent.runs, tt.wantRuns)
}
if len(ledger.finished) != tt.wantFinished {
t.Errorf("claims finished = %d, want %d", len(ledger.finished), tt.wantFinished)
}
})
}
}
func TestTrimGateProgressKeepsTheFailure(t *testing.T) {
in := "go: downloading github.com/a/b v1.0.0\ngo: downloading github.com/c/d v2.0.0\nprotoc-gen-go: executable file not found in $PATH\nFAIL\n"
want := "protoc-gen-go: executable file not found in $PATH\nFAIL\n"
if got := trimGateProgress(in); got != want {
t.Fatalf("trimGateProgress = %q, want %q", got, want)
}
}
@@ -61,6 +61,17 @@ func baselineEnvironmentError(argv []string, out, cause string) error {
strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes))
}
// baselineSharedError is the park reason for a task that did not repair the
// baseline because another task owns that repair.
func baselineSharedError(argv []string, out string, claim BaselineClaim) error {
verdict := fmt.Sprintf("task %d repaired it on its own branch; retry once that change is merged", claim.Task)
if claim.Failure != "" {
verdict = fmt.Sprintf("task %d could not repair it: %s", claim.Task, claim.Failure)
}
return fmt.Errorf("baseline red -- %s fails at this revision and %s\n\ngate output:\n%s",
strings.Join(argv, " "), verdict, clipTail(out, baselineParkOutputBytes))
}
// baselineFixFailedError is the park reason for every failure after the
// baseline repair agent was dispatched. A failure that names only the agent --
// a provider 402, say -- and drops the gate output leaves the operator with no
+3 -2
View File
@@ -37,10 +37,11 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
}
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
cmd.Dir = tc.Dir
out, err := cmd.CombinedOutput()
raw, err := cmd.CombinedOutput()
if err == nil {
continue
}
out := []byte(trimGateProgress(string(raw)))
// An environment failure cannot be repaired in the worktree, so park.
if cause := gateEnvironmentCause(string(out), err); cause != "" {
return baselineEnvironmentError(argv, string(out), cause)
@@ -48,7 +49,7 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error {
tc.Log.WarnContext(ctx, "baseline red -- auto-fixing pre-existing gate failure",
"cmd", strings.Join(argv, " "),
"err", clipTail(string(out), baselineWarnLogBytes))
if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil {
if fixErr := repairBaselineOnce(ctx, tc, argv, out); fixErr != nil {
return fixErr
}
}
+2 -1
View File
@@ -243,7 +243,8 @@ func newGateRepositoryStage(settings yaml.Node) (Stage, error) {
func runGateCommand(ctx context.Context, dir string, command agentrun.Command) (string, error) {
cmd := exec.CommandContext(ctx, command.Argv[0], command.Argv[1:]...)
cmd.Dir = dir
out, err := cmd.CombinedOutput()
raw, err := cmd.CombinedOutput()
out := []byte(trimGateProgress(string(raw)))
name := strings.Join(command.Argv, " ")
switch {
case command.ExpectFailure && err == nil:
+6
View File
@@ -94,6 +94,9 @@ type TaskContext struct {
// to record the step as skipped. It is per-attempt scratch state, reset by
// wrapStep before every step.
skippedWhen string
// baselinePoll is how often a task waits on another's repair; zero uses
// the default.
baselinePoll time.Duration
Task *Task
Repo config.Repo
@@ -102,6 +105,9 @@ type TaskContext struct {
Store Store
Trees Trees
Agent agentrun.Runner
// Baseline makes one task repair a red baseline per repository revision.
// Nil leaves every task to repair its own.
Baseline BaselineLedger
// Deliveries records the run's outward effects instead of performing them.
// Nil performs them; a test run's RecordDeliveries sets it.
Deliveries Deliveries
@@ -148,6 +148,7 @@ func (t *Transport) EventPublisher() agentexec.EventPublisher { return t.conn }
// RemoteTrees is the proxied half of workflow.Trees.
type RemoteTrees interface {
Push(ctx context.Context) error
workflow.BaselineLedger
}
// Forger constructs the identity-scoped forge RPC client.
+140
View File
@@ -0,0 +1,140 @@
package worktreerpc
import (
"context"
"encoding/json"
"errors"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/samcharles93/archie-core/internal/domain/workflow"
"github.com/samcharles93/archie-core/internal/natsrpc"
)
const (
SubjectBaseline = "archie.worktree.baseline"
// A repair that never reports back (its container died) stops blocking
// other tasks after claimTTL; a finished verdict is reused for verdictTTL,
// so a transient failure is retried later.
claimTTL = time.Hour
verdictTTL = 15 * time.Minute
)
type BaselineRequest struct {
Grant string `json:"grant"`
Key string `json:"key"`
Finish bool `json:"finish,omitempty"`
Failure string `json:"failure,omitempty"`
}
type BaselineResponse struct {
natsrpc.Envelope
Lead bool `json:"lead,omitempty"`
Task int64 `json:"task,omitempty"`
Done bool `json:"done,omitempty"`
Failure string `json:"failure,omitempty"`
}
type baselineEntry struct {
task int64
done bool
failure string
at time.Time
}
// BaselineClaims records which task repairs each red baseline. It is memory
// only: after a daemon restart the next task simply claims again.
type BaselineClaims struct {
mu sync.Mutex
entries map[string]baselineEntry
now func() time.Time
}
func NewBaselineClaims() *BaselineClaims {
return &BaselineClaims{entries: map[string]baselineEntry{}, now: time.Now}
}
// Claim makes taskID the repair's owner when nobody holds a live claim for
// key; a retry of the owning task takes it back.
func (b *BaselineClaims) Claim(taskID int64, key string) workflow.BaselineClaim {
b.mu.Lock()
defer b.mu.Unlock()
now := b.now()
e, ok := b.entries[key]
ttl := claimTTL
if e.done {
ttl = verdictTTL
}
if !ok || e.task == taskID || now.Sub(e.at) > ttl {
b.entries[key] = baselineEntry{task: taskID, at: now}
return workflow.BaselineClaim{Lead: true}
}
return workflow.BaselineClaim{Task: e.task, Done: e.done, Failure: e.failure}
}
// Finish records the owner's verdict; failure is empty when it repaired the
// baseline.
func (b *BaselineClaims) Finish(taskID int64, key, failure string) {
b.mu.Lock()
defer b.mu.Unlock()
if e, ok := b.entries[key]; ok && e.task == taskID {
b.entries[key] = baselineEntry{task: taskID, done: true, failure: failure, at: b.now()}
}
}
func (s *Server) handleBaseline(parent context.Context, identity string, msg *nats.Msg) {
var req BaselineRequest
if err := json.Unmarshal(msg.Data, &req); err != nil {
s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)})
return
}
ctx, cancel := s.handlerContext(parent)
defer cancel()
task, err := publishable(ctx, s.Runs, req.Grant, identity)
if err != nil || s.Baselines == nil {
if err == nil {
err = errors.New("baseline claims unavailable")
}
s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)})
return
}
key := task.Owner + "/" + task.Repo + " " + req.Key
if req.Finish {
s.Baselines.Finish(task.ID, key, req.Failure)
s.respondBaseline(msg, BaselineResponse{})
return
}
claim := s.Baselines.Claim(task.ID, key)
s.respondBaseline(msg, BaselineResponse{Lead: claim.Lead, Task: claim.Task, Done: claim.Done, Failure: claim.Failure})
}
// ClaimBaselineFix asks the daemon whether this task repairs the red baseline
// named by key.
func (c *Client) ClaimBaselineFix(ctx context.Context, key string) (workflow.BaselineClaim, error) {
resp, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key})
if err != nil {
return workflow.BaselineClaim{}, err
}
return workflow.BaselineClaim{Lead: resp.Lead, Task: resp.Task, Done: resp.Done, Failure: resp.Failure}, nil
}
// FinishBaselineFix reports the repair's outcome; failure is empty on success.
func (c *Client) FinishBaselineFix(ctx context.Context, key, failure string) error {
_, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key, Finish: true, Failure: failure})
return err
}
func (s *Server) respondBaseline(msg *nats.Msg, resp BaselineResponse) {
natsrpc.Respond(msg, s.Log, "worktreerpc", resp)
}
func (c *Client) baseline(ctx context.Context, req BaselineRequest) (BaselineResponse, error) {
resp, err := natsrpc.Call[BaselineResponse](ctx, c.rpc(), c.subject(SubjectBaseline), req)
if err != nil {
return BaselineResponse{}, err
}
return resp, resp.Err()
}
+12 -9
View File
@@ -77,6 +77,8 @@ type Server struct {
Resolve func(context.Context, *workflow.Task) (*worktree.Manager, error)
Runs Runs
Log *slog.Logger
// Baselines decides which task repairs a red baseline; nil refuses claims.
Baselines *BaselineClaims
// Timeout bounds a single publication. Zero uses
// defaultHandlerTimeout.
Timeout time.Duration
@@ -93,18 +95,19 @@ func (s *Server) handlerContext(parent context.Context) (context.Context, contex
// RegisterFor registers publication under the subject scoped to identity.
func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func(), err error) {
subject := SubjectFor(identity, SubjectPush)
subject, baseline := SubjectFor(identity, SubjectPush), SubjectFor(identity, SubjectBaseline)
if identity == "*" {
subject = "archie.worktree.*.push"
subject, baseline = "archie.worktree.*.push", "archie.worktree.*.baseline"
}
scope := func(msg *nats.Msg) string {
if identity == "*" {
return strings.Split(msg.Subject, ".")[2]
}
return identity
}
return natsrpc.RegisterAll(nc, []natsrpc.Registration{
{Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) {
scoped := identity
if identity == "*" {
scoped = strings.Split(msg.Subject, ".")[2]
}
s.handlePush(ctx, scoped, msg)
}},
{Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) { s.handlePush(ctx, scope(msg), msg) }},
{Subject: baseline, Handler: func(ctx context.Context, msg *nats.Msg) { s.handleBaseline(ctx, scope(msg), msg) }},
})
}