feat(daemon): built-in maintenance jobs and idle mirror pruning

Cycle's maintenance passes become a BuiltinJobs list for the scheduler to
seed as locked schedules. Mirrors use a file lock shared by every Manager
and held across refresh and clone, so pruning cannot race a clone.
This commit is contained in:
sam committed 2026-10-26 00:37:25 +11:00
1 parent e951dbd6cd
commit b87365ad26
5 files changed
+300 -39

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
package daemon
import (
"context"
"time"
"github.com/samcharles93/archie-core/internal/worktree"
)
// BuiltinJob is instance maintenance the daemon defines in code. The
// scheduler shows each as a locked schedule: it can be enabled, disabled,
// retimed and run now, but its work and ID are fixed here.
type BuiltinJob struct {
// ID is stable across releases; the scheduler keys its row on it.
ID string
Name string
// DefaultInterval is the schedule a new installation starts with.
DefaultInterval time.Duration
Run func(ctx context.Context) error
}
// mirrorIdle is how long a repository mirror no task clone borrows from
// survives before pruning; the next task for that repository rebuilds it.
const mirrorIdle = 7 * 24 * time.Hour
// BuiltinJobs lists the daemon's maintenance work. Binding dispatch and
// reaction draining are not here: they turn arrived events into tasks, so
// they stay on the intake tick rather than a schedule's coarser cadence.
func (d *Daemon) BuiltinJobs() []BuiltinJob {
return []BuiltinJob{
{ID: "volume-cleanup", Name: "Remove expired volumes", DefaultInterval: time.Hour, Run: d.cleanupExpiredStorage},
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: logsOwnErrors(d.poll)},
{ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: logsOwnErrors(d.reconcilePRs)},
{ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: logsOwnErrors(d.scanPRReviews)},
{ID: "mirror-prune", Name: "Prune repository mirrors", DefaultInterval: 24 * time.Hour, Run: d.pruneMirrors},
}
}
// logsOwnErrors adapts a pass that reports its per-item failures itself.
func logsOwnErrors(pass func(context.Context)) func(context.Context) error {
return func(ctx context.Context) error {
pass(ctx)
return nil
}
}
func (d *Daemon) pruneMirrors(ctx context.Context) error {
removed, err := worktree.PruneMirrors(ctx, d.Cfg.Get().WorkDir, mirrorIdle, time.Now())
for _, mirror := range removed {
d.Log.InfoContext(ctx, "idle repository mirror removed", "path", mirror)
}
return err
}
+20 -18
View File
@@ -280,7 +280,9 @@ func (d *Daemon) emit(ctx context.Context, e events.Event) {
// Startup runs crash recovery and access verification once.
func (d *Daemon) Startup(ctx context.Context) error {
d.cleanupExpiredStorage(ctx)
if err := d.cleanupExpiredStorage(ctx); err != nil {
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
}
n, err := d.Store.RecoverStale(ctx)
if err != nil {
return err
@@ -463,32 +465,32 @@ func (d *Daemon) RunOnce(ctx context.Context) {
}
}
// Cycle is one poll pass: enqueue newly assigned issues, reconcile open PRs
// and run the binding dispatch loop. Draining queued tasks is the caller's job
// Cycle is one poll pass: run every built-in job, then turn captured events
// and review reactions into tasks. Draining queued tasks is the caller's job
// (drainLoop in Run, drainNATS in RunOnce) so there is one drainer.
func (d *Daemon) Cycle(ctx context.Context) {
d.cleanupExpiredStorage(ctx)
d.poll(ctx)
d.reconcilePRs(ctx)
d.scanPRReviews(ctx)
for _, job := range d.BuiltinJobs() {
if err := job.Run(ctx); err != nil {
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
}
}
d.dispatchBindings(ctx)
d.drainReactions(ctx)
}
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) {
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
if ttl <= 0 {
return
if ttl <= 0 || d.Storage == nil {
return nil
}
if d.Storage != nil {
n, err := d.Storage.CleanupExpired(ctx, ttl)
if err != nil {
d.Log.WarnContext(ctx, "persistent volume cleanup failed", "err", err)
}
if n > 0 {
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
}
n, err := d.Storage.CleanupExpired(ctx, ttl)
if n > 0 {
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
}
if err != nil {
return fmt.Errorf("persistent volume cleanup: %w", err)
}
return nil
}
// bindingDispatchBatchLimit caps how many undispatched captures a single
+125
View File
@@ -0,0 +1,125 @@
package worktree
import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/sys/unix"
)
const mirrorName = ".mirror"
// lockMirror takes the exclusive lock on mirror, which every Manager for the
// same work directory shares because it is a file, not process memory. With
// wait false a held lock fails with unix.EWOULDBLOCK instead of blocking. Taking it
// to clone marks the mirror used, which PruneMirrors measures idleness by.
func lockMirror(mirror string, wait bool) (unlock func(), err error) {
path := mirror + ".lock"
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return nil, fmt.Errorf("create mirror parent: %w", err)
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, fmt.Errorf("open mirror lock: %w", err)
}
how := unix.LOCK_EX
if !wait {
how |= unix.LOCK_NB
}
if err := unix.Flock(int(file.Fd()), how); err != nil {
_ = file.Close()
return nil, fmt.Errorf("lock mirror: %w", err)
}
if wait {
now := time.Now()
_ = os.Chtimes(path, now, now)
}
return func() {
_ = unix.Flock(int(file.Fd()), unix.LOCK_UN)
_ = file.Close()
}, nil
}
// PruneMirrors removes every repository mirror under root that no task clone
// borrows from and that no clone has used for idle. A mirror is in use while
// any other entry sits beside it, since task clones live next to their mirror.
// It returns the mirrors removed.
func PruneMirrors(ctx context.Context, root string, idle time.Duration, now time.Time) ([]string, error) {
root, err := filepath.Abs(root)
if err != nil {
return nil, err
}
var removed []string
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}
if err := ctx.Err(); err != nil {
return err
}
if !entry.IsDir() || path == root {
return nil
}
name := entry.Name()
if name != mirrorName {
// Task clones are deep trees; nothing to find inside one.
if strings.HasPrefix(name, "issue-") || strings.HasPrefix(name, ".") {
return filepath.SkipDir
}
return nil
}
done, err := pruneMirror(path, idle, now)
if err != nil {
return err
}
if done {
removed = append(removed, path)
}
return filepath.SkipDir
})
return removed, err
}
// pruneMirror removes mirror when it is idle and alone in its directory.
func pruneMirror(mirror string, idle time.Duration, now time.Time) (bool, error) {
unlock, err := lockMirror(mirror, false)
if errors.Is(err, unix.EWOULDBLOCK) {
return false, nil // a clone is using it right now
}
if err != nil {
return false, err
}
defer unlock()
siblings, err := os.ReadDir(filepath.Dir(mirror))
if err != nil {
return false, err
}
for _, sibling := range siblings {
if name := sibling.Name(); name != mirrorName && name != mirrorName+".lock" {
return false, nil
}
}
info, err := os.Stat(mirror + ".lock")
if err != nil {
return false, err
}
if now.Sub(info.ModTime()) < idle {
return false, nil
}
// The lock file stays: removing it while held would let a later clone
// lock a new file while another process still holds the old one.
if err := os.RemoveAll(mirror); err != nil {
return false, fmt.Errorf("remove mirror: %w", err)
}
return true, nil
}
+93
View File
@@ -0,0 +1,93 @@
package worktree
import (
"context"
"os"
"path/filepath"
"testing"
"time"
)
func TestPruneMirrorsRemovesOnlyIdleUnusedMirrorsInsideRoot(t *testing.T) {
now := time.Now()
old := now.Add(-48 * time.Hour)
tests := []struct {
name string
setup func(t *testing.T, repo, outside string)
remove bool
}{
{name: "idle and alone", remove: true, setup: func(t *testing.T, repo, _ string) {
mirror(t, repo, old)
}},
{name: "a task clone borrows it", setup: func(t *testing.T, repo, _ string) {
mirror(t, repo, old)
mkdir(t, filepath.Join(repo, "issue-1"))
}},
{name: "used recently", setup: func(t *testing.T, repo, _ string) {
mirror(t, repo, now.Add(-time.Hour))
}},
{name: "a clone holds the lock", setup: func(t *testing.T, repo, _ string) {
mirror(t, repo, old)
unlock, err := lockMirror(filepath.Join(repo, mirrorName), false)
if err != nil {
t.Fatal(err)
}
t.Cleanup(unlock)
}},
{name: "symlink to a directory outside the root", setup: func(t *testing.T, repo, outside string) {
mkdir(t, repo)
if err := os.Symlink(outside, filepath.Join(repo, mirrorName)); err != nil {
t.Fatal(err)
}
lock(t, repo, old)
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
sentinel := filepath.Join(outside, "keep")
if err := os.WriteFile(sentinel, nil, 0o600); err != nil {
t.Fatal(err)
}
repo := filepath.Join(root, "org-x", "owner", "repo")
tt.setup(t, repo, outside)
removed, err := PruneMirrors(context.Background(), root, 24*time.Hour, now)
if err != nil {
t.Fatal(err)
}
_, statErr := os.Lstat(filepath.Join(repo, mirrorName))
if gone := os.IsNotExist(statErr); gone != tt.remove || (len(removed) == 1) != tt.remove {
t.Fatalf("removed %v, mirror gone %v; want removal %v", removed, gone, tt.remove)
}
if _, err := os.Stat(sentinel); err != nil {
t.Fatalf("pruning touched a path outside the root: %v", err)
}
})
}
}
func mirror(t *testing.T, repo string, used time.Time) {
t.Helper()
mkdir(t, filepath.Join(repo, mirrorName, "objects"))
lock(t, repo, used)
}
func lock(t *testing.T, repo string, used time.Time) {
t.Helper()
path := filepath.Join(repo, mirrorName+".lock")
if err := os.WriteFile(path, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chtimes(path, used, used); err != nil {
t.Fatal(err)
}
}
func mkdir(t *testing.T, dir string) {
t.Helper()
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
+9 -21
View File
@@ -14,7 +14,6 @@ import (
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/go-git/go-billy/v6"
@@ -52,10 +51,6 @@ type Manager struct {
// Log optionally receives diagnostics and non-fatal warnings from
// worktree operations.
Log *slog.Logger
// mirrorLocks serialises mirror refreshes per repository directory.
mirrorMu sync.Mutex
mirrorLocks map[string]*sync.Mutex
}
func (m *Manager) logger() *slog.Logger {
@@ -169,6 +164,13 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
return fmt.Errorf("create worktree parent: %w", err)
}
// The lock spans the clone too, so PruneMirrors cannot remove the mirror
// between its refresh and the clone that borrows from it.
unlock, err := lockMirror(m.mirrorDir(owner, repo), true)
if err != nil {
return err
}
defer unlock()
mirror, err := m.ensureMirror(ctx, owner, repo)
if err != nil {
return err
@@ -189,25 +191,11 @@ func (m *Manager) mirrorDir(owner, repo string) string {
return filepath.Join(filepath.Dir(m.Dir(owner, repo, 1)), ".mirror")
}
func (m *Manager) mirrorLock(dir string) *sync.Mutex {
m.mirrorMu.Lock()
defer m.mirrorMu.Unlock()
if m.mirrorLocks == nil {
m.mirrorLocks = map[string]*sync.Mutex{}
}
if m.mirrorLocks[dir] == nil {
m.mirrorLocks[dir] = &sync.Mutex{}
}
return m.mirrorLocks[dir]
}
// ensureMirror brings the repository's shared mirror up to date, creating it
// on first use and recreating it when it tracks a different URL.
// on first use and recreating it when it tracks a different URL. The caller
// holds the mirror lock.
func (m *Manager) ensureMirror(ctx context.Context, owner, repo string) (string, error) {
dir := m.mirrorDir(owner, repo)
lock := m.mirrorLock(dir)
lock.Lock()
defer lock.Unlock()
url := m.cloneURL(owner, repo)
if !repositoryUsesURL(dir, url) {