feat(daemon): built-in maintenance jobs and idle mirror pruning
Cycle's maintenance passes become a BuiltinJobs list for the scheduler to seed as locked schedules. Mirrors use a file lock shared by every Manager and held across refresh and clone, so pruning cannot race a clone.
This commit is contained in:
5 files changed
+300
-39
No files matched your search
@@ -0,0 +1,53 @@
|
||||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/worktree"
|
||||
)
|
||||
|
||||
// BuiltinJob is instance maintenance the daemon defines in code. The
|
||||
// scheduler shows each as a locked schedule: it can be enabled, disabled,
|
||||
// retimed and run now, but its work and ID are fixed here.
|
||||
type BuiltinJob struct {
|
||||
// ID is stable across releases; the scheduler keys its row on it.
|
||||
ID string
|
||||
Name string
|
||||
// DefaultInterval is the schedule a new installation starts with.
|
||||
DefaultInterval time.Duration
|
||||
Run func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// mirrorIdle is how long a repository mirror no task clone borrows from
|
||||
// survives before pruning; the next task for that repository rebuilds it.
|
||||
const mirrorIdle = 7 * 24 * time.Hour
|
||||
|
||||
// BuiltinJobs lists the daemon's maintenance work. Binding dispatch and
|
||||
// reaction draining are not here: they turn arrived events into tasks, so
|
||||
// they stay on the intake tick rather than a schedule's coarser cadence.
|
||||
func (d *Daemon) BuiltinJobs() []BuiltinJob {
|
||||
return []BuiltinJob{
|
||||
{ID: "volume-cleanup", Name: "Remove expired volumes", DefaultInterval: time.Hour, Run: d.cleanupExpiredStorage},
|
||||
{ID: "issue-discovery", Name: "Discover forge issues", DefaultInterval: time.Minute, Run: logsOwnErrors(d.poll)},
|
||||
{ID: "pr-reconcile", Name: "Reconcile pull requests", DefaultInterval: time.Minute, Run: logsOwnErrors(d.reconcilePRs)},
|
||||
{ID: "pr-review-scan", Name: "Scan pull request reviews", DefaultInterval: time.Minute, Run: logsOwnErrors(d.scanPRReviews)},
|
||||
{ID: "mirror-prune", Name: "Prune repository mirrors", DefaultInterval: 24 * time.Hour, Run: d.pruneMirrors},
|
||||
}
|
||||
}
|
||||
|
||||
// logsOwnErrors adapts a pass that reports its per-item failures itself.
|
||||
func logsOwnErrors(pass func(context.Context)) func(context.Context) error {
|
||||
return func(ctx context.Context) error {
|
||||
pass(ctx)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Daemon) pruneMirrors(ctx context.Context) error {
|
||||
removed, err := worktree.PruneMirrors(ctx, d.Cfg.Get().WorkDir, mirrorIdle, time.Now())
|
||||
for _, mirror := range removed {
|
||||
d.Log.InfoContext(ctx, "idle repository mirror removed", "path", mirror)
|
||||
}
|
||||
return err
|
||||
}
|
||||
+20
-18
@@ -280,7 +280,9 @@ func (d *Daemon) emit(ctx context.Context, e events.Event) {
|
||||
|
||||
// Startup runs crash recovery and access verification once.
|
||||
func (d *Daemon) Startup(ctx context.Context) error {
|
||||
d.cleanupExpiredStorage(ctx)
|
||||
if err := d.cleanupExpiredStorage(ctx); err != nil {
|
||||
d.Log.WarnContext(ctx, "startup volume cleanup failed", "err", err)
|
||||
}
|
||||
n, err := d.Store.RecoverStale(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -463,32 +465,32 @@ func (d *Daemon) RunOnce(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Cycle is one poll pass: enqueue newly assigned issues, reconcile open PRs
|
||||
// and run the binding dispatch loop. Draining queued tasks is the caller's job
|
||||
// Cycle is one poll pass: run every built-in job, then turn captured events
|
||||
// and review reactions into tasks. Draining queued tasks is the caller's job
|
||||
// (drainLoop in Run, drainNATS in RunOnce) so there is one drainer.
|
||||
func (d *Daemon) Cycle(ctx context.Context) {
|
||||
d.cleanupExpiredStorage(ctx)
|
||||
d.poll(ctx)
|
||||
d.reconcilePRs(ctx)
|
||||
d.scanPRReviews(ctx)
|
||||
for _, job := range d.BuiltinJobs() {
|
||||
if err := job.Run(ctx); err != nil {
|
||||
d.Log.WarnContext(ctx, "built-in job failed", "job", job.ID, "err", err)
|
||||
}
|
||||
}
|
||||
d.dispatchBindings(ctx)
|
||||
d.drainReactions(ctx)
|
||||
}
|
||||
|
||||
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) {
|
||||
func (d *Daemon) cleanupExpiredStorage(ctx context.Context) error {
|
||||
ttl := d.Cfg.Get().Containers.VolumeTTL.Std()
|
||||
if ttl <= 0 {
|
||||
return
|
||||
if ttl <= 0 || d.Storage == nil {
|
||||
return nil
|
||||
}
|
||||
if d.Storage != nil {
|
||||
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
||||
if err != nil {
|
||||
d.Log.WarnContext(ctx, "persistent volume cleanup failed", "err", err)
|
||||
}
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
|
||||
}
|
||||
n, err := d.Storage.CleanupExpired(ctx, ttl)
|
||||
if n > 0 {
|
||||
d.Log.InfoContext(ctx, "expired persistent volumes removed", "count", n)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("persistent volume cleanup: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bindingDispatchBatchLimit caps how many undispatched captures a single
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package worktree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const mirrorName = ".mirror"
|
||||
|
||||
// lockMirror takes the exclusive lock on mirror, which every Manager for the
|
||||
// same work directory shares because it is a file, not process memory. With
|
||||
// wait false a held lock fails with unix.EWOULDBLOCK instead of blocking. Taking it
|
||||
// to clone marks the mirror used, which PruneMirrors measures idleness by.
|
||||
func lockMirror(mirror string, wait bool) (unlock func(), err error) {
|
||||
path := mirror + ".lock"
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create mirror parent: %w", err)
|
||||
}
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open mirror lock: %w", err)
|
||||
}
|
||||
how := unix.LOCK_EX
|
||||
if !wait {
|
||||
how |= unix.LOCK_NB
|
||||
}
|
||||
if err := unix.Flock(int(file.Fd()), how); err != nil {
|
||||
_ = file.Close()
|
||||
return nil, fmt.Errorf("lock mirror: %w", err)
|
||||
}
|
||||
if wait {
|
||||
now := time.Now()
|
||||
_ = os.Chtimes(path, now, now)
|
||||
}
|
||||
return func() {
|
||||
_ = unix.Flock(int(file.Fd()), unix.LOCK_UN)
|
||||
_ = file.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PruneMirrors removes every repository mirror under root that no task clone
|
||||
// borrows from and that no clone has used for idle. A mirror is in use while
|
||||
// any other entry sits beside it, since task clones live next to their mirror.
|
||||
// It returns the mirrors removed.
|
||||
func PruneMirrors(ctx context.Context, root string, idle time.Duration, now time.Time) ([]string, error) {
|
||||
root, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var removed []string
|
||||
err = filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() || path == root {
|
||||
return nil
|
||||
}
|
||||
name := entry.Name()
|
||||
if name != mirrorName {
|
||||
// Task clones are deep trees; nothing to find inside one.
|
||||
if strings.HasPrefix(name, "issue-") || strings.HasPrefix(name, ".") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
done, err := pruneMirror(path, idle, now)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done {
|
||||
removed = append(removed, path)
|
||||
}
|
||||
return filepath.SkipDir
|
||||
})
|
||||
return removed, err
|
||||
}
|
||||
|
||||
// pruneMirror removes mirror when it is idle and alone in its directory.
|
||||
func pruneMirror(mirror string, idle time.Duration, now time.Time) (bool, error) {
|
||||
unlock, err := lockMirror(mirror, false)
|
||||
if errors.Is(err, unix.EWOULDBLOCK) {
|
||||
return false, nil // a clone is using it right now
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
siblings, err := os.ReadDir(filepath.Dir(mirror))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, sibling := range siblings {
|
||||
if name := sibling.Name(); name != mirrorName && name != mirrorName+".lock" {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
info, err := os.Stat(mirror + ".lock")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if now.Sub(info.ModTime()) < idle {
|
||||
return false, nil
|
||||
}
|
||||
// The lock file stays: removing it while held would let a later clone
|
||||
// lock a new file while another process still holds the old one.
|
||||
if err := os.RemoveAll(mirror); err != nil {
|
||||
return false, fmt.Errorf("remove mirror: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package worktree
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPruneMirrorsRemovesOnlyIdleUnusedMirrorsInsideRoot(t *testing.T) {
|
||||
now := time.Now()
|
||||
old := now.Add(-48 * time.Hour)
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, repo, outside string)
|
||||
remove bool
|
||||
}{
|
||||
{name: "idle and alone", remove: true, setup: func(t *testing.T, repo, _ string) {
|
||||
mirror(t, repo, old)
|
||||
}},
|
||||
{name: "a task clone borrows it", setup: func(t *testing.T, repo, _ string) {
|
||||
mirror(t, repo, old)
|
||||
mkdir(t, filepath.Join(repo, "issue-1"))
|
||||
}},
|
||||
{name: "used recently", setup: func(t *testing.T, repo, _ string) {
|
||||
mirror(t, repo, now.Add(-time.Hour))
|
||||
}},
|
||||
{name: "a clone holds the lock", setup: func(t *testing.T, repo, _ string) {
|
||||
mirror(t, repo, old)
|
||||
unlock, err := lockMirror(filepath.Join(repo, mirrorName), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(unlock)
|
||||
}},
|
||||
{name: "symlink to a directory outside the root", setup: func(t *testing.T, repo, outside string) {
|
||||
mkdir(t, repo)
|
||||
if err := os.Symlink(outside, filepath.Join(repo, mirrorName)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lock(t, repo, old)
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
sentinel := filepath.Join(outside, "keep")
|
||||
if err := os.WriteFile(sentinel, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repo := filepath.Join(root, "org-x", "owner", "repo")
|
||||
tt.setup(t, repo, outside)
|
||||
|
||||
removed, err := PruneMirrors(context.Background(), root, 24*time.Hour, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, statErr := os.Lstat(filepath.Join(repo, mirrorName))
|
||||
if gone := os.IsNotExist(statErr); gone != tt.remove || (len(removed) == 1) != tt.remove {
|
||||
t.Fatalf("removed %v, mirror gone %v; want removal %v", removed, gone, tt.remove)
|
||||
}
|
||||
if _, err := os.Stat(sentinel); err != nil {
|
||||
t.Fatalf("pruning touched a path outside the root: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mirror(t *testing.T, repo string, used time.Time) {
|
||||
t.Helper()
|
||||
mkdir(t, filepath.Join(repo, mirrorName, "objects"))
|
||||
lock(t, repo, used)
|
||||
}
|
||||
|
||||
func lock(t *testing.T, repo string, used time.Time) {
|
||||
t.Helper()
|
||||
path := filepath.Join(repo, mirrorName+".lock")
|
||||
if err := os.WriteFile(path, nil, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Chtimes(path, used, used); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mkdir(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v6"
|
||||
@@ -52,10 +51,6 @@ type Manager struct {
|
||||
// Log optionally receives diagnostics and non-fatal warnings from
|
||||
// worktree operations.
|
||||
Log *slog.Logger
|
||||
|
||||
// mirrorLocks serialises mirror refreshes per repository directory.
|
||||
mirrorMu sync.Mutex
|
||||
mirrorLocks map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
func (m *Manager) logger() *slog.Logger {
|
||||
@@ -169,6 +164,13 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string
|
||||
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||
return fmt.Errorf("create worktree parent: %w", err)
|
||||
}
|
||||
// The lock spans the clone too, so PruneMirrors cannot remove the mirror
|
||||
// between its refresh and the clone that borrows from it.
|
||||
unlock, err := lockMirror(m.mirrorDir(owner, repo), true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
mirror, err := m.ensureMirror(ctx, owner, repo)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -189,25 +191,11 @@ func (m *Manager) mirrorDir(owner, repo string) string {
|
||||
return filepath.Join(filepath.Dir(m.Dir(owner, repo, 1)), ".mirror")
|
||||
}
|
||||
|
||||
func (m *Manager) mirrorLock(dir string) *sync.Mutex {
|
||||
m.mirrorMu.Lock()
|
||||
defer m.mirrorMu.Unlock()
|
||||
if m.mirrorLocks == nil {
|
||||
m.mirrorLocks = map[string]*sync.Mutex{}
|
||||
}
|
||||
if m.mirrorLocks[dir] == nil {
|
||||
m.mirrorLocks[dir] = &sync.Mutex{}
|
||||
}
|
||||
return m.mirrorLocks[dir]
|
||||
}
|
||||
|
||||
// ensureMirror brings the repository's shared mirror up to date, creating it
|
||||
// on first use and recreating it when it tracks a different URL.
|
||||
// on first use and recreating it when it tracks a different URL. The caller
|
||||
// holds the mirror lock.
|
||||
func (m *Manager) ensureMirror(ctx context.Context, owner, repo string) (string, error) {
|
||||
dir := m.mirrorDir(owner, repo)
|
||||
lock := m.mirrorLock(dir)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
url := m.cloneURL(owner, repo)
|
||||
if !repositoryUsesURL(dir, url) {
|
||||
|
||||
Reference in new issue
Block a user