fix(daemon): reclaim stale task clones when the work volume is full; clean up dead tasks too (#1829)

This commit is contained in:
sam committed 2026-11-09 09:41:33 +11:00
1 parent fd14b8d358
commit 055f2da9cd
2 files changed
+67 -1

No files matched your search

+20 -1
View File
@@ -312,6 +312,9 @@ const (
minCloneSpace = 1 << 30
diskCheckEvery = 15 * time.Second
diskWaitLimit = 10 * time.Minute
// cloneRetention is how long an untouched clone, such as a parked task's,
// is kept before a full work volume may reclaim it.
cloneRetention = 72 * time.Hour
)
// sweepAccess accepts pending invitations and verifies push access through
@@ -1814,6 +1817,7 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
}
target = worktree.Target(task.Branch)
}
d.reclaimStaleClones(ctx, task, trees)
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
@@ -1843,6 +1847,21 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
return dir, true
}
// reclaimStaleClones frees the work volume before a clone waits on it, by
// removing the oldest clones nothing has touched for cloneRetention.
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
return
}
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace)
if err != nil {
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
}
if n > 0 {
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
}
}
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
// holds uncommitted work or its state cannot be read.
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
@@ -1864,7 +1883,7 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow
}
// Settled statuses. Keep the worktree while a PR exists.
switch latest.Status {
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted:
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
if latest.PRNumber != 0 {
return
}
+47
View File
@@ -4,8 +4,11 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"sort"
"syscall"
"time"
)
@@ -52,3 +55,47 @@ func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64,
return nil
}
}
// ReclaimStale removes task clones under root that have been untouched for
// longer than minAge, oldest first, until free reports at least floor bytes.
// keep is never removed. It returns how many clones it removed.
func ReclaimStale(root, keep string, minAge time.Duration, free func() (uint64, error), floor uint64) (int, error) {
type clone struct {
dir string
used time.Time
}
var stale []clone
cutoff := time.Now().Add(-minAge)
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil || !d.IsDir() || !issueDir.MatchString(d.Name()) {
return nil //nolint:nilerr // an unreadable branch is skipped, not fatal
}
git, err := os.Stat(filepath.Join(path, ".git"))
if err != nil {
return filepath.SkipDir
}
// The task brief and every git write touch .git, so its mtime is the
// clone's last use.
if path != keep && git.ModTime().Before(cutoff) {
stale = append(stale, clone{path, git.ModTime()})
}
return filepath.SkipDir
})
if err != nil {
return 0, err
}
sort.Slice(stale, func(i, j int) bool { return stale[i].used.Before(stale[j].used) })
removed := 0
for _, c := range stale {
if have, err := free(); err != nil || have >= floor {
return removed, nil
}
if err := os.RemoveAll(c.dir); err != nil {
return removed, err
}
removed++
}
return removed, nil
}
var issueDir = regexp.MustCompile(`^issue-\d+$`)