fix(daemon): reclaim stale task clones when the work volume is full; clean up dead tasks too (#1829)
This commit is contained in:
2 files changed
+67
-1
No files matched your search
@@ -312,6 +312,9 @@ const (
|
|||||||
minCloneSpace = 1 << 30
|
minCloneSpace = 1 << 30
|
||||||
diskCheckEvery = 15 * time.Second
|
diskCheckEvery = 15 * time.Second
|
||||||
diskWaitLimit = 10 * time.Minute
|
diskWaitLimit = 10 * time.Minute
|
||||||
|
// cloneRetention is how long an untouched clone, such as a parked task's,
|
||||||
|
// is kept before a full work volume may reclaim it.
|
||||||
|
cloneRetention = 72 * time.Hour
|
||||||
)
|
)
|
||||||
|
|
||||||
// sweepAccess accepts pending invitations and verifies push access through
|
// sweepAccess accepts pending invitations and verifies push access through
|
||||||
@@ -1814,6 +1817,7 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
|
|||||||
}
|
}
|
||||||
target = worktree.Target(task.Branch)
|
target = worktree.Target(task.Branch)
|
||||||
}
|
}
|
||||||
|
d.reclaimStaleClones(ctx, task, trees)
|
||||||
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
if err := worktree.AwaitSpace(ctx, trees.FreeBytes, minCloneSpace, diskCheckEvery, diskWaitLimit); err != nil {
|
||||||
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
d.Log.WarnContext(ctx, "worktree prepare delayed", "task", task.ID, "err", err)
|
||||||
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
d.parkRunningTask(ctx, task, "worktree prepare failed: "+err.Error(), taskstate.ParkTransient)
|
||||||
@@ -1843,6 +1847,21 @@ func (d *Daemon) prepareWorkspace(ctx context.Context, task *workflow.Task, tree
|
|||||||
return dir, true
|
return dir, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reclaimStaleClones frees the work volume before a clone waits on it, by
|
||||||
|
// removing the oldest clones nothing has touched for cloneRetention.
|
||||||
|
func (d *Daemon) reclaimStaleClones(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||||
|
if have, err := trees.FreeBytes(); err != nil || have >= minCloneSpace {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
n, err := worktree.ReclaimStale(d.Cfg.Get().WorkDir, trees.Dir(task.Owner, task.Repo, task.IssueNumber), cloneRetention, trees.FreeBytes, minCloneSpace)
|
||||||
|
if err != nil {
|
||||||
|
d.Log.WarnContext(ctx, "stale clone reclaim failed", "task", task.ID, "err", err)
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
d.Log.InfoContext(ctx, "reclaimed stale clones on a full work volume", "task", task.ID, "clones", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
|
// cleanupTerminalTaskWorktree removes a settled task's worktree, unless it
|
||||||
// holds uncommitted work or its state cannot be read.
|
// holds uncommitted work or its state cannot be read.
|
||||||
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow.Task, trees *worktree.Manager) {
|
||||||
@@ -1864,7 +1883,7 @@ func (d *Daemon) cleanupTerminalTaskWorktree(ctx context.Context, task *workflow
|
|||||||
}
|
}
|
||||||
// Settled statuses. Keep the worktree while a PR exists.
|
// Settled statuses. Keep the worktree while a PR exists.
|
||||||
switch latest.Status {
|
switch latest.Status {
|
||||||
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted:
|
case workflow.StatusMerged, workflow.StatusRejected, workflow.StatusClosedWontDo, workflow.StatusCompleted, workflow.StatusDead:
|
||||||
if latest.PRNumber != 0 {
|
if latest.PRNumber != 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,8 +4,11 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -52,3 +55,47 @@ func AwaitSpace(ctx context.Context, free func() (uint64, error), floor uint64,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReclaimStale removes task clones under root that have been untouched for
|
||||||
|
// longer than minAge, oldest first, until free reports at least floor bytes.
|
||||||
|
// keep is never removed. It returns how many clones it removed.
|
||||||
|
func ReclaimStale(root, keep string, minAge time.Duration, free func() (uint64, error), floor uint64) (int, error) {
|
||||||
|
type clone struct {
|
||||||
|
dir string
|
||||||
|
used time.Time
|
||||||
|
}
|
||||||
|
var stale []clone
|
||||||
|
cutoff := time.Now().Add(-minAge)
|
||||||
|
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || !d.IsDir() || !issueDir.MatchString(d.Name()) {
|
||||||
|
return nil //nolint:nilerr // an unreadable branch is skipped, not fatal
|
||||||
|
}
|
||||||
|
git, err := os.Stat(filepath.Join(path, ".git"))
|
||||||
|
if err != nil {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
// The task brief and every git write touch .git, so its mtime is the
|
||||||
|
// clone's last use.
|
||||||
|
if path != keep && git.ModTime().Before(cutoff) {
|
||||||
|
stale = append(stale, clone{path, git.ModTime()})
|
||||||
|
}
|
||||||
|
return filepath.SkipDir
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
sort.Slice(stale, func(i, j int) bool { return stale[i].used.Before(stale[j].used) })
|
||||||
|
removed := 0
|
||||||
|
for _, c := range stale {
|
||||||
|
if have, err := free(); err != nil || have >= floor {
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(c.dir); err != nil {
|
||||||
|
return removed, err
|
||||||
|
}
|
||||||
|
removed++
|
||||||
|
}
|
||||||
|
return removed, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var issueDir = regexp.MustCompile(`^issue-\d+$`)
|
||||||
Reference in new issue
Block a user