From 055f2da9cda09b76c810afac99d089dfe35ae651 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 9 Nov 2026 09:33:40 +1100 Subject: [PATCH] fix(daemon): reclaim stale task clones when the work volume is full; clean up dead tasks too (#1829) --- internal/daemon/daemon.go | 21 ++++++++++++++++- internal/worktree/space.go | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 28ab594bf..7f43d9f2a 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -312,6 +312,9 @@ const ( minCloneSpace = 1 << 30 diskCheckEvery = 15 * time.Second diskWaitLimit = 10 * time.Minute + // cloneRetention is how long an untouched clone, such as a parked task's, + // is kept before a full work volume may reclaim it. + cloneRetention = 72 * time.Hour ) // sweepAccess accepts pending invitations and verifies push access through @@ -1814,6 +1817,7 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree } target = worktree.Target(task.Branch) } + d.reclaimStaleClones(ctx, task, trees) if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil { d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err) d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient) @@ -1843,6 +1847,21 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree return dir, true } +// reclaimStaleClones frees the work volume before a clone waits on it, by +// removing the oldest clones nothing has touched for cloneRetention. +func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { + if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace { + return + } + n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace) + if err != nil { + d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err) + } + if n > 0 { + d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n) + } +} + // cleanupTerminalTaskWorktree removes a settled task's worktree, unless it // holds uncommitted work or its state cannot be read. func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) { @@ -1864,7 +1883,7 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow } // Settled statuses. Keep the worktree while a PR exists. switch latest.Status { - case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted: + case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead: if latest.PRNumber != 0 { return } diff --git a/internal/worktree/space.go b/internal/worktree/space.go index 692c29ab2..730fc8e13 100644 --- a/internal/worktree/space.go +++ b/internal/worktree/space.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "io/fs" "os" "path/filepath" + "regexp" + "sort" "syscall" "time" ) @@ -52,3 +55,47 @@ func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64, return nil } } + +// ReclaimStale removes task clones under root that have been untouched for +// longer than minAge, oldest first, until free reports at least floor bytes. +// keep is never removed. It returns how many clones it removed. +func ReclaimStale(root, keep string, minAge time.Duration, free func() (uint64, error), floor uint64) (int, error) { + type clone struct { + dir string + used time.Time + } + var stale []clone + cutoff := time.Now().Add(-minAge) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || !issueDir.MatchString(d.Name()) { + return nil //nolint:nilerr // an unreadable branch is skipped, not fatal + } + git, err := os.Stat(filepath.Join(path, ".git")) + if err != nil { + return filepath.SkipDir + } + // The task brief and every git write touch .git, so its mtime is the + // clone's last use. + if path != keep && git.ModTime().Before(cutoff) { + stale = append(stale, clone{path, git.ModTime()}) + } + return filepath.SkipDir + }) + if err != nil { + return 0, err + } + sort.Slice(stale, func(i, j int) bool { return stale[i].used.Before(stale[j].used) }) + removed := 0 + for _, c := range stale { + if have, err := free(); err != nil || have >= floor { + return removed, nil + } + if err := os.RemoveAll(c.dir); err != nil { + return removed, err + } + removed++ + } + return removed, nil +} + +var issueDir = regexp.MustCompile(`^issue-\d+$`)