Files

477 lines
18 KiB
Go

package daemon
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/workflow"
workflowtask "github.com/samcharles93/archie-core/internal/domain/workflow/task"
"github.com/samcharles93/archie-core/internal/forge"
"github.com/samcharles93/archie-core/internal/storage"
"github.com/samcharles93/archie-core/internal/taskstate"
"github.com/samcharles93/archie-core/internal/worktree"
)
// repoHub is the per-task repository surface the pipeline resolves through.
type repoHub interface {
configFor(task *workflow.Task) config.Config
forgeFor(ctx context.Context, task *workflow.Task) forge.Forge
treesFor(task *workflow.Task) *worktree.Manager
Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error)
}
var _ repoHub = (*Daemon)(nil)
func (d *Daemon) process(ctx context.Context, task *workflow.Task, hold *taskHold) {
// A task off the NATS stream only has an id once the store has claimed its
// row, inside this call, so its capacity is named here.
hold.name(task.ID, task.Attempt)
// The log opens before any check that can park, so every park reason
// reaches the attempt log.
defer d.openTaskLog(task)()
// The workflow may name the identity the run acts as, which everything
// below is chosen by, so it is settled before anything else.
if !d.adoptWorkflowIdentity(ctx, task) {
return
}
// Register first so the task can be stopped during setup.
ctx, finished := d.running.begin(ctx, task.ID, task.Identity)
defer finished()
hold.bindStop(ctx)
if d.parkIfIdentityUnresolvable(ctx, task) {
return
}
trees, repo, err := d.taskWorkspace(ctx, task)
if err != nil {
d.parkBlockedTask(ctx, task, err.Error(), pinParkClass(err), parkBlocker(err))
return
}
if d.Runtime == nil {
d.parkCapabilityUnavailable(ctx, task)
return
}
d.Log.InfoContext(ctx, "processing task", "repo", repo.FullName(), "issue", task.IssueNumber, "attempt", task.Attempt)
workDir, ok := d.prepareWorkspace(ctx, task, trees, repo)
if !ok {
return
}
if !task.HasRepository() {
defer func() {
if err := trees.RemoveScratch(task.ID); err != nil {
d.Log.WarnContext(ctx, "scratch workspace cleanup failed", "task", task.ID, "err", err)
}
}()
}
// The workflow is pinned before the container starts, because the
// profile it names decides the container's image.
profile, ok := d.pinTaskProfile(ctx, task)
if !ok {
return
}
ctr, credential, revokeCredential, ok := d.acquireTaskContainer(ctx, task, repo, workDir, profile)
if !ok {
return
}
defer d.Runtime.Release(ctx, ctr)
defer revokeCredential()
// Run the whole task in archie-agent, which calls back to the daemon for
// store, forge and push operations.
limitCtx, stopLimit := withTaskTimeLimit(ctx, d.configFor(task).Budgets.TaskWallClock.Std())
runCtx, stopWatch := withContainerExit(limitCtx, ctr)
d.runViaAgent(runCtx, task, repo, profile, credential)
stopWatch()
stopLimit()
d.teardownStorage(ctx, task, repo, workDir)
d.cleanupTerminalTaskWorktree(ctx, task, trees)
}
func (d *Daemon) taskWorkspace(ctx context.Context, task *workflow.Task) (*worktree.Manager, config.Repo, error) {
if task.HasRepository() {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err == nil && !task.IsTestRun() && workflow.DefinitionPushes(task.WorkflowDefinitionYAML) {
// Checked before the run rather than at its commit step, so a
// missing permit parks in seconds instead of after the agent's work.
_, err = d.Repository(ctx, task, config.OperationPush)
}
return resolved.Trees, resolved.Repository, err
}
trees := d.treesFor(task)
repo, ok := d.repoFor(task)
if !ok && task.HasRepository() {
return nil, repo, pinFailure{taskstate.ParkTerminal, errors.New("repository is no longer configured")}
}
return trees, repo, nil
}
// prepareWorkspace makes the directory the container binds as its
// workspace, which must exist before Acquire: a fresh clone of the task's
// repository, or an empty scratch directory for a task with none. It parks
// the task and reports false when it cannot.
func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, trees *worktree.Manager, repo config.Repo) (string, bool) {
if !task.HasRepository() {
dir, err := trees.PrepareScratch(task.ID)
if err != nil {
d.Log.ErrorContext(ctx, "scratch workspace prepare failed", "err", err)
d.parkRunningTask(ctx, task, "scratch workspace prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
return dir, true
}
// The retry mode on the task decides whether to resume the pushed branch or
// reset onto base. Prepare clones a missing worktree either way.
target := worktree.Fresh
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryKeepWorktree {
target = worktree.Keep
// Consumed: a later plain requeue must not skip the reset.
task.RetryMode = ""
}
if taskstate.NormalizeRetryMode(task.RetryMode) == taskstate.RetryContinuePushedWork {
if task.Branch == "" {
// The action refuses this combination, so an empty branch here is a
// dispatch bug upstream and no retry can fix it.
d.Log.ErrorContext(ctx, "task has no persisted branch to continue", "task", task.ID)
d.parkRunningTask(ctx, task, "worktree resume failed: task has no branch to continue", taskstate.ParkNeedsDecision)
return "", false
}
target = worktree.Target(task.Branch)
}
d.reclaimStaleClones(ctx, task, trees)
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
return "", false
}
// Every task gets an independent full clone.
prepare := func() (string, string, error) {
return trees.Prepare(ctx, task.Owner, task.Repo, repo.BaseBranch(), task.IssueNumber, task.Title, task.Body, task.Labels, target)
}
dir, branch, err := prepare()
if d.reownDenied(ctx, trees.Dir(task.Owner, task.Repo, task.IssueNumber), err) {
dir, branch, err = prepare()
}
if err != nil {
reason := "worktree prepare failed: " + err.Error()
if target != worktree.Fresh {
reason = "worktree resume failed: " + err.Error()
}
d.Log.ErrorContext(ctx, "worktree prepare failed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, reason, taskstate.ParkTransient)
return "", false
}
task.Branch = branch
if err := d.Store.Update(ctx, task); err != nil {
d.Log.WarnContext(ctx, "task branch not persisted", "task", task.ID, "err", err)
}
return dir, true
}
// reclaimStaleClones frees the work volume before a clone waits on it, by
// removing the oldest clones nothing has touched for cloneRetention. A clone
// with uncommitted or unpushed work is kept.
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
return
}
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace, func(dir string) bool { return trees.HoldsUnsharedWork(ctx, dir) })
if err != nil {
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
}
if n > 0 {
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
}
}
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
// holds uncommitted work or its state cannot be read.
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if trees == nil || task == nil || !task.HasRepository() {
return
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
latest, err := d.Store.TaskByID(cleanupCtx, task.ID)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup task lookup failed", "task", task.ID, "err", err)
return
}
// A run that handed off shares its work item's worktree with its
// successor, which owns it from here.
if latest == nil || latest.HandoffToTaskID != 0 {
return
}
// Settled statuses. Keep the worktree while a PR exists.
switch latest.Status {
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
if latest.PRNumber != 0 {
return
}
if d.worktreeHoldsUncapturedWork(cleanupCtx, trees, task) {
return
}
if err := d.cleanupWorktree(cleanupCtx, trees, task.Owner, task.Repo, task.IssueNumber); err != nil {
d.Log.WarnContext(ctx, "terminal worktree cleanup failed", "task", task.ID, "err", err)
}
}
}
// worktreeHoldsUncapturedWork reports whether a terminal task's worktree has
// to survive cleanup because it may hold work no commit captured. A worktree
// that is not on disk at all has nothing to keep, so a task whose clone was
// already removed still cleans up whatever leftovers share its path.
func (d *Daemon) worktreeHoldsUncapturedWork(ctx context.Context, trees *worktree.Manager, task *workflow.Task) bool {
dir := trees.Dir(task.Owner, task.Repo, task.IssueNumber)
if _, err := os.Stat(dir); err != nil {
if errors.Is(err, fs.ErrNotExist) {
return false
}
d.Log.WarnContext(ctx, "terminal worktree kept: it could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
uncommitted, err := trees.HasUncommittedChanges(ctx, dir)
if err != nil {
d.Log.WarnContext(ctx, "terminal worktree kept: uncommitted work could not be read; archive the task to remove it", "task", task.ID, "dir", dir, "err", err)
return true
}
if uncommitted {
d.Log.InfoContext(ctx, "terminal worktree kept: it holds uncommitted work; archive the task to remove it", "task", task.ID, "dir", dir)
return true
}
return false
}
// teardownStorage runs after the workflow completes. The Docker backend is
// a no-op; future backends (temp volumes, NFS leases) use this hook.
func (d *Daemon) teardownStorage(ctx context.Context, task *workflow.Task, repo config.Repo, workDir string) {
if d.Storage == nil {
return
}
if err := d.Storage.Teardown(ctx, storage.TaskRef{
WorktreeDir: workDir,
Ecosystem: repo.Ecosystem,
PersistentStorage: repo.PersistentStorage,
Owner: task.Owner,
Repo: task.Repo,
}); err != nil {
d.Log.WarnContext(ctx, "storage teardown failed", "task", task.ID, "err", err)
}
}
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
if ttl <= 0 || d.Storage == nil {
return nil
}
n, err := d.Storage.CleanupExpired(ctx, ttl)
if n > 0 {
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
}
if err != nil {
return fmt.Errorf("persistent volume cleanup: %w", err)
}
return nil
}
// forgeFor returns the forge client that owns task: its organisation
// connection, else the root d.Forge for the root alone. This is the safety
// boundary that keeps one identity's forge token from being used against
// another identity's repos.
func (d *Daemon) forgeFor(ctx context.Context, task *workflow.Task) forge.Forge {
if task.HasRepository() {
resolved, err := d.Repository(ctx, task, config.OperationClone)
if err != nil {
d.Log.WarnContext(ctx, "no forge for task", "task", task.ID, "repo", task.Owner+"/"+task.Repo, "org", task.Org, "err", err)
return forge.NewNoop(d.Log)
}
return resolved.Forge
}
if !d.rootTask(task) {
return forge.NewNoop(d.Log)
}
return d.Forge
}
// treesFor returns the worktree manager that owns task, mirroring forgeFor.
func (d *Daemon) treesFor(task *workflow.Task) *worktree.Manager {
if task.HasRepository() {
// Cleanup needs only the org's path, not a still-valid external credential.
return &worktree.Manager{WorkDir: worktree.OrgWorkDir(d.Cfg.Get().WorkDir, task.OrgOrDefault())}
}
if !d.rootTask(task) {
return &worktree.Manager{WorkDir: filepath.Join(d.Cfg.Get().WorkDir, "identity-"+string(identity.StableID(task.Identity)))}
}
return d.Trees
}
// repoFor resolves a root task's repo config from Cfg.Repos.
func (d *Daemon) repoFor(t *workflow.Task) (config.Repo, bool) {
if !d.rootTask(t) {
return config.Repo{}, false
}
for _, r := range d.Cfg.Get().Repos {
if r.Owner == t.Owner && r.Name == t.Repo {
return r, true
}
}
return config.Repo{}, false
}
func (d *Daemon) rootTask(task *workflow.Task) bool {
return identity.IdentityID(task.Identity) == identity.SystemID
}
func (d *Daemon) configFor(task *workflow.Task) config.Config {
cfg := d.Cfg.Get()
if !d.rootTask(task) {
cfg.BotUser, cfg.BotEmail, cfg.Forge = "", "", config.Forge{}
cfg.Org, cfg.GrantedCredentials, cfg.Identities, cfg.IdentityGrants = task.OrgOrDefault(), nil, nil, nil
cfg.Repos, cfg.Models, cfg.Providers = nil, nil, nil
}
return cfg
}
// allowConcurrentForTask reports whether the task's repo allows concurrent
// tasks. Unknown repos do not.
func (d *Daemon) allowConcurrentForTask(task *workflow.Task) bool {
if !task.HasRepository() {
return true
}
repo, ok := d.repoFor(task)
return ok && repo.AllowConcurrent
}
// Repository resolves current credentials and permissions for the run's
// execution identity, never the identity that originally requested the task.
func (d *Daemon) Repository(ctx context.Context, task *workflow.Task, operation string) (RepositoryAccess, error) {
run := *task
if run.WorkflowDefinitionYAML != "" {
iface, err := workflowtask.ParseWorkflowInterface(run.WorkflowDefinitionYAML)
if err != nil || iface.Identity == "" {
return RepositoryAccess{}, fmt.Errorf("workflow execution identity unavailable")
}
_, name, err := d.Identity.declaredIdentity(ctx, &run, iface.Identity)
if err != nil {
return RepositoryAccess{}, err
}
run.Identity = name
}
ctx, err := d.Identity.workflowContext(ctx, &run)
if err != nil {
return RepositoryAccess{}, err
}
if _, err := d.Identity.databaseExecutionIdentity(ctx, &run, identity.IdentityID(run.Identity)); err != nil {
return RepositoryAccess{}, err
}
return d.ResolveRepository(ctx, &run, operation)
}
// botUserForTask resolves the bot login that owns task, so the reaction
// consumer can exclude Archie's own comments: the global forge bot user for
// root tasks.
func (d *Daemon) botUserForTask(task *workflowtask.Task) string {
if !d.rootTask(task) {
return ""
}
return d.Cfg.Get().BotUser
}
// RemoveWorktree deletes a task's clone from the trees of the identity that
// owns it. Archiving calls it: terminal cleanup keeps a worktree that may hold
// uncaptured work, and the operator's archive is the decision to discard it.
func (d *Daemon) RemoveWorktree(task *workflow.Task) error {
trees := d.treesFor(task)
if trees == nil {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
return d.cleanupWorktree(ctx, trees, task.Owner, task.Repo, task.IssueNumber)
}
// cleanupWorktree removes a task's clone, first handing back any root-owned
// files a killed container left in it.
func (d *Daemon) cleanupWorktree(ctx context.Context, trees *worktree.Manager, owner, repo string, issue int) error {
err := trees.Cleanup(owner, repo, issue)
if d.reownDenied(ctx, trees.Dir(owner, repo, issue), err) {
return trees.Cleanup(owner, repo, issue)
}
return err
}
// reownDenied repairs dir when err says the daemon was refused access to it,
// and reports whether the caller should retry. A SIGKILLed task container
// leaves root-owned files that only a container can chown back.
func (d *Daemon) reownDenied(ctx context.Context, dir string, err error) bool {
if err == nil || !errors.Is(err, fs.ErrPermission) || d.Runtime == nil {
return false
}
if rerr := d.Runtime.Reown(ctx, dir, os.Getuid(), os.Getgid()); rerr != nil {
d.Log.WarnContext(ctx, "worktree ownership repair failed", "dir", dir, "err", rerr)
return false
}
return true
}
// A clone waits, holding its slot, until the work volume has minCloneSpace
// free, and parks as transient if it does not within diskWaitLimit. Clones
// share objects with the repository's mirror, so the floor covers a checkout
// and its build output, not a full clone.
const (
minCloneSpace = 1 << 30
diskCheckEvery = 15 * time.Second
diskWaitLimit = 10 * time.Minute
// cloneRetention is how long an untouched clone, such as a parked task's,
// is kept before a full work volume may reclaim it.
cloneRetention = 72 * time.Hour
)
// withContainerExit derives a context cancelled when exited closes. A dead
// container never answers its taskrun request, so without this the request
// blocks forever and the task row stays running.
func withContainerExit(ctx context.Context, ctr TaskContainer) (context.Context, func()) {
runCtx, cancel := context.WithCancelCause(ctx)
go func() {
select {
case <-ctr.Exited():
if detail := ctr.ExitDetail(); detail != "" {
cancel(fmt.Errorf("%w: %s", errContainerExited, detail))
return
}
cancel(errContainerExited)
case <-runCtx.Done():
}
}()
return runCtx, func() { cancel(nil) }
}
// errContainerExited is the cancellation cause when a task's agent container
// stops before answering its taskrun request.
var errContainerExited = errors.New("agent container exited before the task finished")
// withTaskTimeLimit bounds a whole task run. A limit of zero or less applies
// no bound.
func withTaskTimeLimit(ctx context.Context, limit time.Duration) (context.Context, func()) {
if limit <= 0 {
return ctx, func() {}
}
return context.WithTimeoutCause(ctx, limit, fmt.Errorf("%w (%s)", errTaskTimeLimit, limit))
}
// errTaskTimeLimit is the cancellation cause when a task outruns its overall
// time limit (the execution settings' max task runtime).
var errTaskTimeLimit = errors.New("task exceeded its time limit")