diff --git a/internal/app/agentworker/task_execution.go b/internal/app/agentworker/task_execution.go index 6cc45e0b4..3bf8ee79f 100644 --- a/internal/app/agentworker/task_execution.go +++ b/internal/app/agentworker/task_execution.go @@ -33,6 +33,7 @@ import ( // worktreerpc; local git operations run on the bind-mounted worktree. type remoteTrees interface { Push(ctx context.Context) error + workflow.BaselineLedger } type hybridTrees struct { @@ -263,6 +264,9 @@ func runTask(ctx context.Context, req taskrun.Request, dependencies taskDependen Log: log, Guardrails: dependencies.guardrails, } + if dependencies.trees != nil { + tc.Baseline = dependencies.trees + } tc.Instructions = req.Instructions configureTaskContext(tc, workDir) diff --git a/internal/app/archied/main.go b/internal/app/archied/main.go index 766d6ab15..e1d0a168c 100644 --- a/internal/app/archied/main.go +++ b/internal/app/archied/main.go @@ -362,7 +362,7 @@ func registerTaskRPCServers(nc *natsio.Conn, resolve func(context.Context, *work resolved, err := resolve(ctx, t, config.OperationPR) return resolved.Forge, err }} - trees := &worktreerpc.Server{Runs: runs, Log: log, Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) { + trees := &worktreerpc.Server{Runs: runs, Log: log, Baselines: worktreerpc.NewBaselineClaims(), Resolve: func(ctx context.Context, t *workflow.Task) (*worktree.Manager, error) { resolved, err := resolve(ctx, t, config.OperationPush) return resolved.Trees, err }} diff --git a/internal/app/archied/task_broker.go b/internal/app/archied/task_broker.go index 90fde30fa..33134a335 100644 --- a/internal/app/archied/task_broker.go +++ b/internal/app/archied/task_broker.go @@ -35,6 +35,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s agentexec.SubjectForEvents(taskID), spanrelay.Subject(taskID), worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush), + worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectBaseline), messagerpc.SubjectFor(task.Identity), }, forgerpc.Subjects(task.Identity)...) return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil diff --git a/internal/domain/workflow/baseline_ledger.go b/internal/domain/workflow/baseline_ledger.go new file mode 100644 index 000000000..f678d0dc7 --- /dev/null +++ b/internal/domain/workflow/baseline_ledger.go @@ -0,0 +1,95 @@ +package workflow + +import ( + "context" + "fmt" + "os/exec" + "regexp" + "strings" + "time" +) + +// BaselineClaim answers a task's request to repair a red baseline. +type BaselineClaim struct { + // Lead is true when the caller owns the repair and must run it. + Lead bool + // Task is the task that owns the repair when Lead is false. + Task int64 + // Done is true once that task has finished; Failure is its failure, empty + // when it repaired the baseline on its own branch. + Done bool + Failure string +} + +// BaselineLedger makes one task repair a red baseline per repository revision +// and command. Every task runs in its own container, so the ledger lives with +// the daemon. +type BaselineLedger interface { + ClaimBaselineFix(ctx context.Context, key string) (BaselineClaim, error) + FinishBaselineFix(ctx context.Context, key, failure string) error +} + +const defaultBaselinePoll = 15 * time.Second + +// repairBaselineOnce runs the repair when this task leads it, or reports the +// verdict of the task that does. Without a ledger every task repairs its own. +func repairBaselineOnce(ctx context.Context, tc *TaskContext, argv []string, out []byte) error { + if tc.Baseline == nil { + return runBaselineFix(ctx, tc, argv, out) + } + revision, err := exec.CommandContext(ctx, "git", "-C", tc.Dir, "rev-parse", "HEAD").Output() + if err != nil { + return fmt.Errorf("resolve baseline revision: %w", err) + } + key := strings.TrimSpace(string(revision)) + " " + strings.Join(argv, " ") + poll := tc.baselinePoll + if poll <= 0 { + poll = defaultBaselinePoll + } + for { + claim, err := tc.Baseline.ClaimBaselineFix(ctx, key) + if err != nil { + return fmt.Errorf("claim baseline fix: %w", err) + } + switch { + case claim.Lead: + fixErr := runBaselineFix(ctx, tc, argv, out) + failure := "" + if fixErr != nil { + failure = clip(fixErr.Error(), baselineParkOutputBytes) + } + if err := tc.Baseline.FinishBaselineFix(context.WithoutCancel(ctx), key, failure); err != nil { + tc.Log.WarnContext(ctx, "release baseline fix claim", "err", err) + } + return fixErr + case !claim.Done: + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(poll): + } + default: + return baselineSharedError(argv, string(out), claim) + } + } +} + +// gateProgressLine matches the package-download chatter `go` prints before the +// real failure. +var gateProgressLine = regexp.MustCompile(`^go: (downloading|finding|extracting) `) + +// trimGateProgress drops download progress from gate output so the failing +// command's own lines lead. +func trimGateProgress(out string) string { + if !strings.Contains(out, "go: ") { + return out + } + lines := strings.Split(out, "\n") + kept := lines[:0] + for _, line := range lines { + if !gateProgressLine.MatchString(line) { + kept = append(kept, line) + } + } + return strings.Join(kept, "\n") +} diff --git a/internal/domain/workflow/baseline_ledger_test.go b/internal/domain/workflow/baseline_ledger_test.go new file mode 100644 index 000000000..954a9e170 --- /dev/null +++ b/internal/domain/workflow/baseline_ledger_test.go @@ -0,0 +1,110 @@ +package workflow + +import ( + "context" + "log/slog" + "os/exec" + "strings" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/config" + "github.com/samcharles93/archie-core/internal/domain/agentrun" + "github.com/samcharles93/archie-core/internal/domain/workflow/task" +) + +type countingRunner struct{ runs int } + +func (r *countingRunner) Run(context.Context, string, agentrun.Request, agentrun.Reporter) (agentrun.Result, error) { + r.runs++ + return agentrun.Result{Version: agentrun.ProtocolVersion, Status: "failed"}, nil +} + +// scriptedLedger answers claims from a fixed list, then repeats the last one. +type scriptedLedger struct { + claims []BaselineClaim + finished []string +} + +func (l *scriptedLedger) ClaimBaselineFix(context.Context, string) (BaselineClaim, error) { + claim := l.claims[0] + if len(l.claims) > 1 { + l.claims = l.claims[1:] + } + return claim, nil +} + +func (l *scriptedLedger) FinishBaselineFix(_ context.Context, _, failure string) error { + l.finished = append(l.finished, failure) + return nil +} + +func TestRedBaselineIsRepairedByOneTask(t *testing.T) { + tests := []struct { + name string + claims []BaselineClaim + wantRuns int + wantFinished int + wantInErr string + }{ + { + name: "the task that leads repairs it", + claims: []BaselineClaim{{Lead: true}}, + wantRuns: 1, + wantFinished: 1, + wantInErr: "did not repair it", + }, + { + name: "a task waits on the lead, then reports its failure", + claims: []BaselineClaim{{Task: 7}, {Task: 7}, {Task: 7, Done: true, Failure: "builder status failed"}}, + wantInErr: "task 7 could not repair it: builder status failed", + }, + { + name: "a task reuses a repair made on another branch", + claims: []BaselineClaim{{Task: 7, Done: true}}, + wantInErr: "task 7 repaired it on its own branch", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + for _, args := range [][]string{{"init", "-q"}, {"-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "base"}} { + if out, err := exec.CommandContext(t.Context(), "git", append([]string{"-C", dir}, args...)...).CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + agent := &countingRunner{} + ledger := &scriptedLedger{claims: tt.claims} + tc := &TaskContext{ + Task: &task.Task{ID: 1, Attempt: 1, Source: task.SourceTest}, + Repo: config.Repo{Owner: "acme", Name: "widgets"}, + Cfg: config.Config{Models: map[string]string{"default": "m"}}, + Dir: dir, + Log: slog.Default(), + Agent: agent, + Baseline: ledger, + baselinePoll: time.Millisecond, + } + + err := repairBaselineOnce(context.Background(), tc, []string{"task", "check"}, []byte("FAIL")) + + if err == nil || !strings.Contains(err.Error(), tt.wantInErr) { + t.Fatalf("err = %v, want it to contain %q", err, tt.wantInErr) + } + if agent.runs != tt.wantRuns { + t.Errorf("fix agents run = %d, want %d", agent.runs, tt.wantRuns) + } + if len(ledger.finished) != tt.wantFinished { + t.Errorf("claims finished = %d, want %d", len(ledger.finished), tt.wantFinished) + } + }) + } +} + +func TestTrimGateProgressKeepsTheFailure(t *testing.T) { + in := "go: downloading github.com/a/b v1.0.0\ngo: downloading github.com/c/d v2.0.0\nprotoc-gen-go: executable file not found in $PATH\nFAIL\n" + want := "protoc-gen-go: executable file not found in $PATH\nFAIL\n" + if got := trimGateProgress(in); got != want { + t.Fatalf("trimGateProgress = %q, want %q", got, want) + } +} diff --git a/internal/domain/workflow/gate_environment.go b/internal/domain/workflow/gate_environment.go index f04fc590c..012b5b381 100644 --- a/internal/domain/workflow/gate_environment.go +++ b/internal/domain/workflow/gate_environment.go @@ -61,6 +61,17 @@ func baselineEnvironmentError(argv []string, out, cause string) error { strings.Join(argv, " "), cause, clipTail(out, baselineParkOutputBytes)) } +// baselineSharedError is the park reason for a task that did not repair the +// baseline because another task owns that repair. +func baselineSharedError(argv []string, out string, claim BaselineClaim) error { + verdict := fmt.Sprintf("task %d repaired it on its own branch; retry once that change is merged", claim.Task) + if claim.Failure != "" { + verdict = fmt.Sprintf("task %d could not repair it: %s", claim.Task, claim.Failure) + } + return fmt.Errorf("baseline red -- %s fails at this revision and %s\n\ngate output:\n%s", + strings.Join(argv, " "), verdict, clipTail(out, baselineParkOutputBytes)) +} + // baselineFixFailedError is the park reason for every failure after the // baseline repair agent was dispatched. A failure that names only the agent -- // a provider 402, say -- and drops the gate output leaves the operator with no diff --git a/internal/domain/workflow/implement.go b/internal/domain/workflow/implement.go index d90132133..0370299a7 100644 --- a/internal/domain/workflow/implement.go +++ b/internal/domain/workflow/implement.go @@ -37,10 +37,11 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error { } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) cmd.Dir = tc.Dir - out, err := cmd.CombinedOutput() + raw, err := cmd.CombinedOutput() if err == nil { continue } + out := []byte(trimGateProgress(string(raw))) // An environment failure cannot be repaired in the worktree, so park. if cause := gateEnvironmentCause(string(out), err); cause != "" { return baselineEnvironmentError(argv, string(out), cause) @@ -48,7 +49,7 @@ func stageBaselineGateRun(ctx context.Context, tc *TaskContext) error { tc.Log.WarnContext(ctx, "baseline red -- auto-fixing pre-existing gate failure", "cmd", strings.Join(argv, " "), "err", clipTail(string(out), baselineWarnLogBytes)) - if fixErr := runBaselineFix(ctx, tc, argv, out); fixErr != nil { + if fixErr := repairBaselineOnce(ctx, tc, argv, out); fixErr != nil { return fixErr } } diff --git a/internal/domain/workflow/repo_steps.go b/internal/domain/workflow/repo_steps.go index 6b7699cdd..8d5b746cc 100644 --- a/internal/domain/workflow/repo_steps.go +++ b/internal/domain/workflow/repo_steps.go @@ -243,7 +243,8 @@ func newGateRepositoryStage(settings yaml.Node) (Stage, error) { func runGateCommand(ctx context.Context, dir string, command agentrun.Command) (string, error) { cmd := exec.CommandContext(ctx, command.Argv[0], command.Argv[1:]...) cmd.Dir = dir - out, err := cmd.CombinedOutput() + raw, err := cmd.CombinedOutput() + out := []byte(trimGateProgress(string(raw))) name := strings.Join(command.Argv, " ") switch { case command.ExpectFailure && err == nil: diff --git a/internal/domain/workflow/workflow.go b/internal/domain/workflow/workflow.go index 04e4454bc..30ad80c2f 100644 --- a/internal/domain/workflow/workflow.go +++ b/internal/domain/workflow/workflow.go @@ -94,6 +94,9 @@ type TaskContext struct { // to record the step as skipped. It is per-attempt scratch state, reset by // wrapStep before every step. skippedWhen string + // baselinePoll is how often a task waits on another's repair; zero uses + // the default. + baselinePoll time.Duration Task *Task Repo config.Repo @@ -102,6 +105,9 @@ type TaskContext struct { Store Store Trees Trees Agent agentrun.Runner + // Baseline makes one task repair a red baseline per repository revision. + // Nil leaves every task to repair its own. + Baseline BaselineLedger // Deliveries records the run's outward effects instead of performing them. // Nil performs them; a test run's RecordDeliveries sets it. Deliveries Deliveries diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index ecef9375c..c46636f72 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -148,6 +148,7 @@ func (t *Transport) EventPublisher() agentexec.EventPublisher { return t.conn } // RemoteTrees is the proxied half of workflow.Trees. type RemoteTrees interface { Push(ctx context.Context) error + workflow.BaselineLedger } // Forger constructs the identity-scoped forge RPC client. diff --git a/internal/worktreerpc/baseline.go b/internal/worktreerpc/baseline.go new file mode 100644 index 000000000..c8c5ce759 --- /dev/null +++ b/internal/worktreerpc/baseline.go @@ -0,0 +1,140 @@ +package worktreerpc + +import ( + "context" + "encoding/json" + "errors" + "sync" + "time" + + "github.com/nats-io/nats.go" + + "github.com/samcharles93/archie-core/internal/domain/workflow" + "github.com/samcharles93/archie-core/internal/natsrpc" +) + +const ( + SubjectBaseline = "archie.worktree.baseline" + + // A repair that never reports back (its container died) stops blocking + // other tasks after claimTTL; a finished verdict is reused for verdictTTL, + // so a transient failure is retried later. + claimTTL = time.Hour + verdictTTL = 15 * time.Minute +) + +type BaselineRequest struct { + Grant string `json:"grant"` + Key string `json:"key"` + Finish bool `json:"finish,omitempty"` + Failure string `json:"failure,omitempty"` +} + +type BaselineResponse struct { + natsrpc.Envelope + Lead bool `json:"lead,omitempty"` + Task int64 `json:"task,omitempty"` + Done bool `json:"done,omitempty"` + Failure string `json:"failure,omitempty"` +} + +type baselineEntry struct { + task int64 + done bool + failure string + at time.Time +} + +// BaselineClaims records which task repairs each red baseline. It is memory +// only: after a daemon restart the next task simply claims again. +type BaselineClaims struct { + mu sync.Mutex + entries map[string]baselineEntry + now func() time.Time +} + +func NewBaselineClaims() *BaselineClaims { + return &BaselineClaims{entries: map[string]baselineEntry{}, now: time.Now} +} + +// Claim makes taskID the repair's owner when nobody holds a live claim for +// key; a retry of the owning task takes it back. +func (b *BaselineClaims) Claim(taskID int64, key string) workflow.BaselineClaim { + b.mu.Lock() + defer b.mu.Unlock() + now := b.now() + e, ok := b.entries[key] + ttl := claimTTL + if e.done { + ttl = verdictTTL + } + if !ok || e.task == taskID || now.Sub(e.at) > ttl { + b.entries[key] = baselineEntry{task: taskID, at: now} + return workflow.BaselineClaim{Lead: true} + } + return workflow.BaselineClaim{Task: e.task, Done: e.done, Failure: e.failure} +} + +// Finish records the owner's verdict; failure is empty when it repaired the +// baseline. +func (b *BaselineClaims) Finish(taskID int64, key, failure string) { + b.mu.Lock() + defer b.mu.Unlock() + if e, ok := b.entries[key]; ok && e.task == taskID { + b.entries[key] = baselineEntry{task: taskID, done: true, failure: failure, at: b.now()} + } +} + +func (s *Server) handleBaseline(parent context.Context, identity string, msg *nats.Msg) { + var req BaselineRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)}) + return + } + ctx, cancel := s.handlerContext(parent) + defer cancel() + task, err := publishable(ctx, s.Runs, req.Grant, identity) + if err != nil || s.Baselines == nil { + if err == nil { + err = errors.New("baseline claims unavailable") + } + s.respondBaseline(msg, BaselineResponse{Envelope: natsrpc.NewEnvelope(err)}) + return + } + key := task.Owner + "/" + task.Repo + " " + req.Key + if req.Finish { + s.Baselines.Finish(task.ID, key, req.Failure) + s.respondBaseline(msg, BaselineResponse{}) + return + } + claim := s.Baselines.Claim(task.ID, key) + s.respondBaseline(msg, BaselineResponse{Lead: claim.Lead, Task: claim.Task, Done: claim.Done, Failure: claim.Failure}) +} + +// ClaimBaselineFix asks the daemon whether this task repairs the red baseline +// named by key. +func (c *Client) ClaimBaselineFix(ctx context.Context, key string) (workflow.BaselineClaim, error) { + resp, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key}) + if err != nil { + return workflow.BaselineClaim{}, err + } + return workflow.BaselineClaim{Lead: resp.Lead, Task: resp.Task, Done: resp.Done, Failure: resp.Failure}, nil +} + +// FinishBaselineFix reports the repair's outcome; failure is empty on success. +func (c *Client) FinishBaselineFix(ctx context.Context, key, failure string) error { + _, err := c.baseline(ctx, BaselineRequest{Grant: c.Grant, Key: key, Finish: true, Failure: failure}) + return err +} + +func (s *Server) respondBaseline(msg *nats.Msg, resp BaselineResponse) { + natsrpc.Respond(msg, s.Log, "worktreerpc", resp) +} + +func (c *Client) baseline(ctx context.Context, req BaselineRequest) (BaselineResponse, error) { + resp, err := natsrpc.Call[BaselineResponse](ctx, c.rpc(), c.subject(SubjectBaseline), req) + if err != nil { + return BaselineResponse{}, err + } + return resp, resp.Err() +} diff --git a/internal/worktreerpc/worktreerpc.go b/internal/worktreerpc/worktreerpc.go index 89c561e6e..62a1c0495 100644 --- a/internal/worktreerpc/worktreerpc.go +++ b/internal/worktreerpc/worktreerpc.go @@ -77,6 +77,8 @@ type Server struct { Resolve func(context.Context, *workflow.Task) (*worktree.Manager, error) Runs Runs Log *slog.Logger + // Baselines decides which task repairs a red baseline; nil refuses claims. + Baselines *BaselineClaims // Timeout bounds a single publication. Zero uses // defaultHandlerTimeout. Timeout time.Duration @@ -93,18 +95,19 @@ func (s *Server) handlerContext(parent context.Context) (context.Context, contex // RegisterFor registers publication under the subject scoped to identity. func (s *Server) RegisterFor(nc *nats.Conn, identity string) (unsubscribe func(), err error) { - subject := SubjectFor(identity, SubjectPush) + subject, baseline := SubjectFor(identity, SubjectPush), SubjectFor(identity, SubjectBaseline) if identity == "*" { - subject = "archie.worktree.*.push" + subject, baseline = "archie.worktree.*.push", "archie.worktree.*.baseline" + } + scope := func(msg *nats.Msg) string { + if identity == "*" { + return strings.Split(msg.Subject, ".")[2] + } + return identity } return natsrpc.RegisterAll(nc, []natsrpc.Registration{ - {Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) { - scoped := identity - if identity == "*" { - scoped = strings.Split(msg.Subject, ".")[2] - } - s.handlePush(ctx, scoped, msg) - }}, + {Subject: subject, Handler: func(ctx context.Context, msg *nats.Msg) { s.handlePush(ctx, scope(msg), msg) }}, + {Subject: baseline, Handler: func(ctx context.Context, msg *nats.Msg) { s.handleBaseline(ctx, scope(msg), msg) }}, }) }