fix(daemon): reclaim stale task clones when the work volume is full; clean up dead tasks too (#1829)
This commit is contained in:
2 files changed
+67
-1
No files matched your search
@@ -312,6 +312,9 @@ const (
|
||||
minCloneSpace = 1 << 30
|
||||
diskCheckEvery = 15 * time.Second
|
||||
diskWaitLimit = 10 * time.Minute
|
||||
// cloneRetention is how long an untouched clone, such as a parked task's,
|
||||
// is kept before a full work volume may reclaim it.
|
||||
cloneRetention = 72 * time.Hour
|
||||
)
|
||||
|
||||
// sweepAccess accepts pending invitations and verifies push access through
|
||||
@@ -1814,6 +1817,7 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
|
||||
}
|
||||
target = worktree.Target(task.Branch)
|
||||
}
|
||||
d.reclaimStaleClones(ctx, task, trees)
|
||||
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
||||
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
||||
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
||||
@@ -1843,6 +1847,21 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
|
||||
return dir, true
|
||||
}
|
||||
|
||||
// reclaimStaleClones frees the work volume before a clone waits on it, by
|
||||
// removing the oldest clones nothing has touched for cloneRetention.
|
||||
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
|
||||
return
|
||||
}
|
||||
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
|
||||
}
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
|
||||
// holds uncommitted work or its state cannot be read.
|
||||
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||
@@ -1864,7 +1883,7 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow
|
||||
}
|
||||
// Settled statuses. Keep the worktree while a PR exists.
|
||||
switch latest.Status {
|
||||
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted:
|
||||
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
|
||||
if latest.PRNumber != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,8 +4,11 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
@@ -52,3 +55,47 @@ func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64,
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ReclaimStale removes task clones under root that have been untouched for
|
||||
// longer than minAge, oldest first, until free reports at least floor bytes.
|
||||
// keep is never removed. It returns how many clones it removed.
|
||||
func ReclaimStale(root, keep string, minAge time.Duration, free func() (uint64, error), floor uint64) (int, error) {
|
||||
type clone struct {
|
||||
dir string
|
||||
used time.Time
|
||||
}
|
||||
var stale []clone
|
||||
cutoff := time.Now().Add(-minAge)
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || !d.IsDir() || !issueDir.MatchString(d.Name()) {
|
||||
return nil //nolint:nilerr // an unreadable branch is skipped, not fatal
|
||||
}
|
||||
git, err := os.Stat(filepath.Join(path, ".git"))
|
||||
if err != nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
// The task brief and every git write touch .git, so its mtime is the
|
||||
// clone's last use.
|
||||
if path != keep && git.ModTime().Before(cutoff) {
|
||||
stale = append(stale, clone{path, git.ModTime()})
|
||||
}
|
||||
return filepath.SkipDir
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
sort.Slice(stale, func(i, j int) bool { return stale[i].used.Before(stale[j].used) })
|
||||
removed := 0
|
||||
for _, c := range stale {
|
||||
if have, err := free(); err != nil || have >= floor {
|
||||
return removed, nil
|
||||
}
|
||||
if err := os.RemoveAll(c.dir); err != nil {
|
||||
return removed, err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
var issueDir = regexp.MustCompile(`^issue-\d+$`)
|
||||
Reference in new issue
Block a user