477 lines
18 KiB
Go
477 lines
18 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
"github.com/samcharles93/archie-core/internal/domain/identity"
|
|
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
|
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
|
|
"github.com/samcharles93/archie-core/internal/forge"
|
|
"github.com/samcharles93/archie-core/internal/storage"
|
|
"github.com/samcharles93/archie-core/internal/taskstate"
|
|
"github.com/samcharles93/archie-core/internal/worktree"
|
|
)
|
|
|
|
// repoHub is the per-task repository surface the pipeline resolves through.
|
|
type repoHub interface {
|
|
configFor(task *workflow.Task) config.Config
|
|
forgeFor(ctx context.Context, task *workflow.Task) forge.Forge
|
|
treesFor(task *workflow.Task) *worktree.Manager
|
|
Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error)
|
|
}
|
|
|
|
var _ repoHub = (*Daemon)(nil)
|
|
|
|
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
|
|
// A task off the NATS stream only has an id once the store has claimed its
|
|
// row, inside this call, so its capacity is named here.
|
|
hold.name(task.ID, task.Attempt)
|
|
// The log opens before any check that can park, so every park reason
|
|
// reaches the attempt log.
|
|
defer d.openTaskLog(task)()
|
|
// The workflow may name the identity the run acts as, which everything
|
|
// below is chosen by, so it is settled before anything else.
|
|
if !d.adoptWorkflowIdentity(ctx, task) {
|
|
return
|
|
}
|
|
// Register first so the task can be stopped during setup.
|
|
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
|
|
defer finished()
|
|
hold.bindStop(ctx)
|
|
|
|
if d.parkIfIdentityUnresolvable(ctx, task) {
|
|
return
|
|
}
|
|
|
|
trees, repo, err := d.taskWorkspace(ctx, task)
|
|
if err != nil {
|
|
d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err))
|
|
return
|
|
}
|
|
|
|
if d.Runtime == nil {
|
|
d.parkCapabilityUnavailable(ctx, task)
|
|
return
|
|
}
|
|
|
|
d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
|
|
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !task.HasRepository() {
|
|
defer func() {
|
|
if err := trees.RemoveScratch(task.ID); err != nil {
|
|
d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// The workflow is pinned before the container starts, because the
|
|
// profile it names decides the container's image.
|
|
profile, ok := d.pinTaskProfile(ctx, task)
|
|
if !ok {
|
|
return
|
|
}
|
|
ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile)
|
|
if !ok {
|
|
return
|
|
}
|
|
defer d.Runtime.Release(ctx, ctr)
|
|
defer revokeCredential()
|
|
|
|
// Run the whole task in archie-agent, which calls back to the daemon for
|
|
// store, forge and push operations.
|
|
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
|
|
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
|
|
d.runViaAgent(runCtx, task, repo, profile, credential)
|
|
stopWatch()
|
|
stopLimit()
|
|
|
|
d.teardownStorage(ctx, task, repo, workDir)
|
|
d.cleanupTerminalTaskWorktree(ctx, task, trees)
|
|
}
|
|
|
|
func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) {
|
|
if task.HasRepository() {
|
|
resolved, err := d.Repository(ctx, task, config.OperationClone)
|
|
if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) {
|
|
// Checked before the run rather than at its commit step, so a
|
|
// missing permit parks in seconds instead of after the agent's work.
|
|
_, err = d.Repository(ctx, task, config.OperationPush)
|
|
}
|
|
return resolved.Trees, resolved.Repository, err
|
|
}
|
|
trees := d.treesFor(task)
|
|
repo, ok := d.repoFor(task)
|
|
if !ok && task.HasRepository() {
|
|
return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")}
|
|
}
|
|
return trees, repo, nil
|
|
}
|
|
|
|
// prepareWorkspace makes the directory the container binds as its
|
|
// workspace, which must exist before Acquire: a fresh clone of the task's
|
|
// repository, or an empty scratch directory for a task with none. It parks
|
|
// the task and reports false when it cannot.
|
|
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
|
|
if !task.HasRepository() {
|
|
dir, err := trees.PrepareScratch(task.ID)
|
|
if err != nil {
|
|
d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err)
|
|
d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
|
|
return "", false
|
|
}
|
|
return dir, true
|
|
}
|
|
// The retry mode on the task decides whether to resume the pushed branch or
|
|
// reset onto base. Prepare clones a missing worktree either way.
|
|
target := worktree.Fresh
|
|
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree {
|
|
target = worktree.Keep
|
|
// Consumed: a later plain requeue must not skip the reset.
|
|
task.RetryMode = ""
|
|
}
|
|
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork {
|
|
if task.Branch == "" {
|
|
// The action refuses this combination, so an empty branch here is a
|
|
// dispatch bug upstream and no retry can fix it.
|
|
d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID)
|
|
d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision)
|
|
return "", false
|
|
}
|
|
target = worktree.Target(task.Branch)
|
|
}
|
|
d.reclaimStaleClones(ctx, task, trees)
|
|
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
|
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
|
return "", false
|
|
}
|
|
// Every task gets an independent full clone.
|
|
prepare := func() (string, string, error) {
|
|
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
|
|
}
|
|
dir, branch, err := prepare()
|
|
if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) {
|
|
dir, branch, err = prepare()
|
|
}
|
|
if err != nil {
|
|
reason := "worktree prepare failed: " + err.Error()
|
|
if target != worktree.Fresh {
|
|
reason = "worktree resume failed: " + err.Error()
|
|
}
|
|
d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err)
|
|
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
|
|
return "", false
|
|
}
|
|
task.Branch = branch
|
|
if err := d.Store.Update(ctx, task); err != nil {
|
|
d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err)
|
|
}
|
|
return dir, true
|
|
}
|
|
|
|
// reclaimStaleClones frees the work volume before a clone waits on it, by
|
|
// removing the oldest clones nothing has touched for cloneRetention. A clone
|
|
// with uncommitted or unpushed work is kept.
|
|
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
|
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
|
|
return
|
|
}
|
|
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) })
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
|
|
}
|
|
if n > 0 {
|
|
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
|
|
}
|
|
}
|
|
|
|
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
|
|
// holds uncommitted work or its state cannot be read.
|
|
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
|
if trees == nil || task == nil || !task.HasRepository() {
|
|
return
|
|
}
|
|
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
|
defer cancel()
|
|
|
|
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
|
|
return
|
|
}
|
|
// A run that handed off shares its work item's worktree with its
|
|
// successor, which owns it from here.
|
|
if latest == nil || latest.HandoffToTaskID != 0 {
|
|
return
|
|
}
|
|
// Settled statuses. Keep the worktree while a PR exists.
|
|
switch latest.Status {
|
|
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
|
|
if latest.PRNumber != 0 {
|
|
return
|
|
}
|
|
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
|
|
return
|
|
}
|
|
if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil {
|
|
d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
|
|
// to survive cleanup because it may hold work no commit captured. A worktree
|
|
// that is not on disk at all has nothing to keep, so a task whose clone was
|
|
// already removed still cleans up whatever leftovers share its path.
|
|
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
|
|
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
|
|
if _, err := os.Stat(dir); err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return false
|
|
}
|
|
d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
|
return true
|
|
}
|
|
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
|
|
return true
|
|
}
|
|
if uncommitted {
|
|
d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// teardownStorage runs after the workflow completes. The Docker backend is
|
|
// a no-op; future backends (temp volumes, NFS leases) use this hook.
|
|
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
|
|
if d.Storage == nil {
|
|
return
|
|
}
|
|
if err := d.Storage.Teardown(ctx, storage.TaskRef{
|
|
WorktreeDir: workDir,
|
|
Ecosystem: repo.Ecosystem,
|
|
PersistentStorage: repo.PersistentStorage,
|
|
Owner: task.Owner,
|
|
Repo: task.Repo,
|
|
}); err != nil {
|
|
d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err)
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
|
|
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
|
|
if ttl <= 0 || d.Storage == nil {
|
|
return nil
|
|
}
|
|
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
|
if n > 0 {
|
|
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
|
|
}
|
|
if err != nil {
|
|
return fmt.Errorf("persistent volume cleanup: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// forgeFor returns the forge client that owns task: its organisation
|
|
// connection, else the root d.Forge for the root alone. This is the safety
|
|
// boundary that keeps one identity's forge token from being used against
|
|
// another identity's repos.
|
|
func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge {
|
|
if task.HasRepository() {
|
|
resolved, err := d.Repository(ctx, task, config.OperationClone)
|
|
if err != nil {
|
|
d.Log.WarnContext(ctx, "no forge for task", "task", task.ID, "repo", task.Owner+"/"+task.Repo, "org", task.Org, "err", err)
|
|
return forge.NewNoop(d.Log)
|
|
}
|
|
return resolved.Forge
|
|
}
|
|
if !d.rootTask(task) {
|
|
return forge.NewNoop(d.Log)
|
|
}
|
|
return d.Forge
|
|
}
|
|
|
|
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
|
|
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
|
|
if task.HasRepository() {
|
|
// Cleanup needs only the org's path, not a still-valid external credential.
|
|
return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())}
|
|
}
|
|
if !d.rootTask(task) {
|
|
return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))}
|
|
}
|
|
return d.Trees
|
|
}
|
|
|
|
// repoFor resolves a root task's repo config from Cfg.Repos.
|
|
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
|
|
if !d.rootTask(t) {
|
|
return config.Repo{}, false
|
|
}
|
|
for _, r := range d.Cfg.Get().Repos {
|
|
if r.Owner == t.Owner && r.Name == t.Repo {
|
|
return r, true
|
|
}
|
|
}
|
|
return config.Repo{}, false
|
|
}
|
|
|
|
func (d *Daemon) rootTask(task *workflow.Task) bool {
|
|
return identity.IdentityID(task.Identity) == identity.SystemID
|
|
}
|
|
|
|
func (d *Daemon) configFor(task *workflow.Task) config.Config {
|
|
cfg := d.Cfg.Get()
|
|
if !d.rootTask(task) {
|
|
cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{}
|
|
cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil
|
|
cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// allowConcurrentForTask reports whether the task's repo allows concurrent
|
|
// tasks. Unknown repos do not.
|
|
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
|
|
if !task.HasRepository() {
|
|
return true
|
|
}
|
|
repo, ok := d.repoFor(task)
|
|
return ok && repo.AllowConcurrent
|
|
}
|
|
|
|
// Repository resolves current credentials and permissions for the run's
|
|
// execution identity, never the identity that originally requested the task.
|
|
func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) {
|
|
run := *task
|
|
if run.WorkflowDefinitionYAML != "" {
|
|
iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML)
|
|
if err != nil || iface.Identity == "" {
|
|
return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable")
|
|
}
|
|
_, name, err := d.Identity.declaredIdentity(ctx, &run, iface.Identity)
|
|
if err != nil {
|
|
return RepositoryAccess{}, err
|
|
}
|
|
run.Identity = name
|
|
}
|
|
ctx, err := d.Identity.workflowContext(ctx, &run)
|
|
if err != nil {
|
|
return RepositoryAccess{}, err
|
|
}
|
|
if _, err := d.Identity.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil {
|
|
return RepositoryAccess{}, err
|
|
}
|
|
return d.ResolveRepository(ctx, &run, operation)
|
|
}
|
|
|
|
// botUserForTask resolves the bot login that owns task, so the reaction
|
|
// consumer can exclude Archie's own comments: the global forge bot user for
|
|
// root tasks.
|
|
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
|
|
if !d.rootTask(task) {
|
|
return ""
|
|
}
|
|
return d.Cfg.Get().BotUser
|
|
}
|
|
|
|
// RemoveWorktree deletes a task's clone from the trees of the identity that
|
|
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
|
|
// uncaptured work, and the operator's archive is the decision to discard it.
|
|
func (d *Daemon) RemoveWorktree(task *workflow.Task) error {
|
|
trees := d.treesFor(task)
|
|
if trees == nil {
|
|
return nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber)
|
|
}
|
|
|
|
// cleanupWorktree removes a task's clone, first handing back any root-owned
|
|
// files a killed container left in it.
|
|
func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error {
|
|
err := trees.Cleanup(owner, repo, issue)
|
|
if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) {
|
|
return trees.Cleanup(owner, repo, issue)
|
|
}
|
|
return err
|
|
}
|
|
|
|
// reownDenied repairs dir when err says the daemon was refused access to it,
|
|
// and reports whether the caller should retry. A SIGKILLed task container
|
|
// leaves root-owned files that only a container can chown back.
|
|
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
|
|
if err == nil || !errors.Is(err, fs.ErrPermission) || d.Runtime == nil {
|
|
return false
|
|
}
|
|
if rerr := d.Runtime.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
|
|
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// A clone waits, holding its slot, until the work volume has minCloneSpace
|
|
// free, and parks as transient if it does not within diskWaitLimit. Clones
|
|
// share objects with the repository's mirror, so the floor covers a checkout
|
|
// and its build output, not a full clone.
|
|
const (
|
|
minCloneSpace = 1 << 30
|
|
diskCheckEvery = 15 * time.Second
|
|
diskWaitLimit = 10 * time.Minute
|
|
// cloneRetention is how long an untouched clone, such as a parked task's,
|
|
// is kept before a full work volume may reclaim it.
|
|
cloneRetention = 72 * time.Hour
|
|
)
|
|
|
|
// withContainerExit derives a context cancelled when exited closes. A dead
|
|
// container never answers its taskrun request, so without this the request
|
|
// blocks forever and the task row stays running.
|
|
func withContainerExit(ctx context.Context, ctr TaskContainer) (context.Context, func()) {
|
|
runCtx, cancel := context.WithCancelCause(ctx)
|
|
go func() {
|
|
select {
|
|
case <-ctr.Exited():
|
|
if detail := ctr.ExitDetail(); detail != "" {
|
|
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
|
|
return
|
|
}
|
|
cancel(errContainerExited)
|
|
case <-runCtx.Done():
|
|
}
|
|
}()
|
|
return runCtx, func() { cancel(nil) }
|
|
}
|
|
|
|
// errContainerExited is the cancellation cause when a task's agent container
|
|
// stops before answering its taskrun request.
|
|
var errContainerExited = errors.New("agent container exited before the task finished")
|
|
|
|
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
|
|
// no bound.
|
|
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
|
|
if limit <= 0 {
|
|
return ctx, func() {}
|
|
}
|
|
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
|
|
}
|
|
|
|
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
|
|
// time limit (the execution settings' max task runtime).
|
|
var errTaskTimeLimit = errors.New("task exceeded its time limit")
|