From 423df0320f749817111e85fabbfb40bf41a3f44d Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 26 Oct 2026 00:37:25 +1100 Subject: [PATCH] feat(daemon): built-in maintenance jobs and idle mirror pruning Cycle's maintenance passes become a BuiltinJobs list for the scheduler to seed as locked schedules. Mirrors use a file lock shared by every Manager and held across refresh and clone, so pruning cannot race a clone. --- internal/daemon/builtin_jobs.go | 53 +++++++++++++ internal/daemon/daemon.go | 38 +++++----- internal/worktree/mirror.go | 125 +++++++++++++++++++++++++++++++ internal/worktree/mirror_test.go | 93 +++++++++++++++++++++++ internal/worktree/worktree.go | 30 +++----- 5 files changed, 300 insertions(+), 39 deletions(-) create mode 100644 internal/daemon/builtin_jobs.go create mode 100644 internal/worktree/mirror.go create mode 100644 internal/worktree/mirror_test.go diff --git a/internal/daemon/builtin_jobs.go b/internal/daemon/builtin_jobs.go new file mode 100644 index 000000000..6418feced --- /dev/null +++ b/internal/daemon/builtin_jobs.go @@ -0,0 +1,53 @@ +package daemon + +import ( + "context" + "time" + + "github.com/samcharles93/archie-core/internal/worktree" +) + +// BuiltinJob is instance maintenance the daemon defines in code. The +// scheduler shows each as a locked schedule: it can be enabled, disabled, +// retimed and run now, but its work and ID are fixed here. +type BuiltinJob struct { + // ID is stable across releases; the scheduler keys its row on it. + ID string + Name string + // DefaultInterval is the schedule a new installation starts with. + DefaultInterval time.Duration + Run func(ctx context.Context) error +} + +// mirrorIdle is how long a repository mirror no task clone borrows from +// survives before pruning; the next task for that repository rebuilds it. +const mirrorIdle = 7 * 24 * time.Hour + +// BuiltinJobs lists the daemon's maintenance work. Binding dispatch and +// reaction draining are not here: they turn arrived events into tasks, so +// they stay on the intake tick rather than a schedule's coarser cadence. +func (d *Daemon) BuiltinJobs() []BuiltinJob { + return []BuiltinJob{ + {ID: "volume-cleanup", Name: "Remove expired volumes", DefaultInterval: time.Hour, Run: d.cleanupExpiredStorage}, + {ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: logsOwnErrors(d.poll)}, + {ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: logsOwnErrors(d.reconcilePRs)}, + {ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: logsOwnErrors(d.scanPRReviews)}, + {ID: "mirror-prune", Name: "Prune repository mirrors", DefaultInterval: 24 * time.Hour, Run: d.pruneMirrors}, + } +} + +// logsOwnErrors adapts a pass that reports its per-item failures itself. +func logsOwnErrors(pass func(context.Context)) func(context.Context) error { + return func(ctx context.Context) error { + pass(ctx) + return nil + } +} + +func (d *Daemon) pruneMirrors(ctx context.Context) error { + removed, err := worktree.PruneMirrors(ctx, d.Cfg.Get().WorkDir, mirrorIdle, time.Now()) + for _, mirror := range removed { + d.Log.InfoContext(ctx, "idle repository mirror removed", "path", mirror) + } + return err +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 823b189b0..a5c2c564e 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -280,7 +280,9 @@ func (d *Daemon) emit(ctx context.Context, e events.Event) { // Startup runs crash recovery and access verification once. func (d *Daemon) Startup(ctx context.Context) error { - d.cleanupExpiredStorage(ctx) + if err := d.cleanupExpiredStorage(ctx); err != nil { + d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err) + } n, err := d.Store.RecoverStale(ctx) if err != nil { return err @@ -463,32 +465,32 @@ func (d *Daemon) RunOnce(ctx context.Context) { } } -// Cycle is one poll pass: enqueue newly assigned issues, reconcile open PRs -// and run the binding dispatch loop. Draining queued tasks is the caller's job +// Cycle is one poll pass: run every built-in job, then turn captured events +// and review reactions into tasks. Draining queued tasks is the caller's job // (drainLoop in Run, drainNATS in RunOnce) so there is one drainer. func (d *Daemon) Cycle(ctx context.Context) { - d.cleanupExpiredStorage(ctx) - d.poll(ctx) - d.reconcilePRs(ctx) - d.scanPRReviews(ctx) + for _, job := range d.BuiltinJobs() { + if err := job.Run(ctx); err != nil { + d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err) + } + } d.dispatchBindings(ctx) d.drainReactions(ctx) } -func (d *Daemon) cleanupExpiredStorage(ctx context.Context) { +func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error { ttl := d.Cfg.Get().Containers.VolumeTTL.Std() - if ttl <= 0 { - return + if ttl <= 0 || d.Storage == nil { + return nil } - if d.Storage != nil { - n, err := d.Storage.CleanupExpired(ctx, ttl) - if err != nil { - d.Log.WarnContext(ctx, "persistent volume cleanup failed", "err", err) - } - if n > 0 { - d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n) - } + n, err := d.Storage.CleanupExpired(ctx, ttl) + if n > 0 { + d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n) } + if err != nil { + return fmt.Errorf("persistent volume cleanup: %w", err) + } + return nil } // bindingDispatchBatchLimit caps how many undispatched captures a single diff --git a/internal/worktree/mirror.go b/internal/worktree/mirror.go new file mode 100644 index 000000000..4115d6cad --- /dev/null +++ b/internal/worktree/mirror.go @@ -0,0 +1,125 @@ +package worktree + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +const mirrorName = ".mirror" + +// lockMirror takes the exclusive lock on mirror, which every Manager for the +// same work directory shares because it is a file, not process memory. With +// wait false a held lock fails with unix.EWOULDBLOCK instead of blocking. Taking it +// to clone marks the mirror used, which PruneMirrors measures idleness by. +func lockMirror(mirror string, wait bool) (unlock func(), err error) { + path := mirror + ".lock" + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("create mirror parent: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open mirror lock: %w", err) + } + how := unix.LOCK_EX + if !wait { + how |= unix.LOCK_NB + } + if err := unix.Flock(int(file.Fd()), how); err != nil { + _ = file.Close() + return nil, fmt.Errorf("lock mirror: %w", err) + } + if wait { + now := time.Now() + _ = os.Chtimes(path, now, now) + } + return func() { + _ = unix.Flock(int(file.Fd()), unix.LOCK_UN) + _ = file.Close() + }, nil +} + +// PruneMirrors removes every repository mirror under root that no task clone +// borrows from and that no clone has used for idle. A mirror is in use while +// any other entry sits beside it, since task clones live next to their mirror. +// It returns the mirrors removed. +func PruneMirrors(ctx context.Context, root string, idle time.Duration, now time.Time) ([]string, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + var removed []string + err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if err := ctx.Err(); err != nil { + return err + } + if !entry.IsDir() || path == root { + return nil + } + name := entry.Name() + if name != mirrorName { + // Task clones are deep trees; nothing to find inside one. + if strings.HasPrefix(name, "issue-") || strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + return nil + } + done, err := pruneMirror(path, idle, now) + if err != nil { + return err + } + if done { + removed = append(removed, path) + } + return filepath.SkipDir + }) + return removed, err +} + +// pruneMirror removes mirror when it is idle and alone in its directory. +func pruneMirror(mirror string, idle time.Duration, now time.Time) (bool, error) { + unlock, err := lockMirror(mirror, false) + if errors.Is(err, unix.EWOULDBLOCK) { + return false, nil // a clone is using it right now + } + if err != nil { + return false, err + } + defer unlock() + + siblings, err := os.ReadDir(filepath.Dir(mirror)) + if err != nil { + return false, err + } + for _, sibling := range siblings { + if name := sibling.Name(); name != mirrorName && name != mirrorName+".lock" { + return false, nil + } + } + info, err := os.Stat(mirror + ".lock") + if err != nil { + return false, err + } + if now.Sub(info.ModTime()) < idle { + return false, nil + } + // The lock file stays: removing it while held would let a later clone + // lock a new file while another process still holds the old one. + if err := os.RemoveAll(mirror); err != nil { + return false, fmt.Errorf("remove mirror: %w", err) + } + return true, nil +} diff --git a/internal/worktree/mirror_test.go b/internal/worktree/mirror_test.go new file mode 100644 index 000000000..b55a0345e --- /dev/null +++ b/internal/worktree/mirror_test.go @@ -0,0 +1,93 @@ +package worktree + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" +) + +func TestPruneMirrorsRemovesOnlyIdleUnusedMirrorsInsideRoot(t *testing.T) { + now := time.Now() + old := now.Add(-48 * time.Hour) + tests := []struct { + name string + setup func(t *testing.T, repo, outside string) + remove bool + }{ + {name: "idle and alone", remove: true, setup: func(t *testing.T, repo, _ string) { + mirror(t, repo, old) + }}, + {name: "a task clone borrows it", setup: func(t *testing.T, repo, _ string) { + mirror(t, repo, old) + mkdir(t, filepath.Join(repo, "issue-1")) + }}, + {name: "used recently", setup: func(t *testing.T, repo, _ string) { + mirror(t, repo, now.Add(-time.Hour)) + }}, + {name: "a clone holds the lock", setup: func(t *testing.T, repo, _ string) { + mirror(t, repo, old) + unlock, err := lockMirror(filepath.Join(repo, mirrorName), false) + if err != nil { + t.Fatal(err) + } + t.Cleanup(unlock) + }}, + {name: "symlink to a directory outside the root", setup: func(t *testing.T, repo, outside string) { + mkdir(t, repo) + if err := os.Symlink(outside, filepath.Join(repo, mirrorName)); err != nil { + t.Fatal(err) + } + lock(t, repo, old) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + sentinel := filepath.Join(outside, "keep") + if err := os.WriteFile(sentinel, nil, 0o600); err != nil { + t.Fatal(err) + } + repo := filepath.Join(root, "org-x", "owner", "repo") + tt.setup(t, repo, outside) + + removed, err := PruneMirrors(context.Background(), root, 24*time.Hour, now) + if err != nil { + t.Fatal(err) + } + _, statErr := os.Lstat(filepath.Join(repo, mirrorName)) + if gone := os.IsNotExist(statErr); gone != tt.remove || (len(removed) == 1) != tt.remove { + t.Fatalf("removed %v, mirror gone %v; want removal %v", removed, gone, tt.remove) + } + if _, err := os.Stat(sentinel); err != nil { + t.Fatalf("pruning touched a path outside the root: %v", err) + } + }) + } +} + +func mirror(t *testing.T, repo string, used time.Time) { + t.Helper() + mkdir(t, filepath.Join(repo, mirrorName, "objects")) + lock(t, repo, used) +} + +func lock(t *testing.T, repo string, used time.Time) { + t.Helper() + path := filepath.Join(repo, mirrorName+".lock") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, used, used); err != nil { + t.Fatal(err) + } +} + +func mkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } +} diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go index 2684fe663..b228cb483 100644 --- a/internal/worktree/worktree.go +++ b/internal/worktree/worktree.go @@ -14,7 +14,6 @@ import ( "path/filepath" "slices" "strings" - "sync" "time" "github.com/go-git/go-billy/v6" @@ -52,10 +51,6 @@ type Manager struct { // Log optionally receives diagnostics and non-fatal warnings from // worktree operations. Log *slog.Logger - - // mirrorLocks serialises mirror refreshes per repository directory. - mirrorMu sync.Mutex - mirrorLocks map[string]*sync.Mutex } func (m *Manager) logger() *slog.Logger { @@ -169,6 +164,13 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { return fmt.Errorf("create worktree parent: %w", err) } + // The lock spans the clone too, so PruneMirrors cannot remove the mirror + // between its refresh and the clone that borrows from it. + unlock, err := lockMirror(m.mirrorDir(owner, repo), true) + if err != nil { + return err + } + defer unlock() mirror, err := m.ensureMirror(ctx, owner, repo) if err != nil { return err @@ -189,25 +191,11 @@ func (m *Manager) mirrorDir(owner, repo string) string { return filepath.Join(filepath.Dir(m.Dir(owner, repo, 1)), ".mirror") } -func (m *Manager) mirrorLock(dir string) *sync.Mutex { - m.mirrorMu.Lock() - defer m.mirrorMu.Unlock() - if m.mirrorLocks == nil { - m.mirrorLocks = map[string]*sync.Mutex{} - } - if m.mirrorLocks[dir] == nil { - m.mirrorLocks[dir] = &sync.Mutex{} - } - return m.mirrorLocks[dir] -} - // ensureMirror brings the repository's shared mirror up to date, creating it -// on first use and recreating it when it tracks a different URL. +// on first use and recreating it when it tracks a different URL. The caller +// holds the mirror lock. func (m *Manager) ensureMirror(ctx context.Context, owner, repo string) (string, error) { dir := m.mirrorDir(owner, repo) - lock := m.mirrorLock(dir) - lock.Lock() - defer lock.Unlock() url := m.cloneURL(owner, repo) if !repositoryUsesURL(dir, url) {