From 6ffbf0e2a2d9483096431102e23731c0eb301a44 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 25 Oct 2026 21:54:50 +1100 Subject: [PATCH] perf(worktree): clone tasks from a shared per-repository mirror --- internal/worktree/http_test.go | 7 +- internal/worktree/worktree.go | 206 ++++++++++++++++++++++++++++++--- 2 files changed, 194 insertions(+), 19 deletions(-) diff --git a/internal/worktree/http_test.go b/internal/worktree/http_test.go index 941884043..ca3d2f147 100644 --- a/internal/worktree/http_test.go +++ b/internal/worktree/http_test.go @@ -108,9 +108,10 @@ func TestForgeHTTPAuthentication(t *testing.T) { backend.ServeHTTP(w, r) })) defer server.Close() - m := *manager - m.BaseURL = server.URL - m.WorkDir = t.TempDir() + m := &Manager{ + BotUser: manager.BotUser, BotEmail: manager.BotEmail, Token: manager.Token, + BaseURL: server.URL, WorkDir: t.TempDir(), + } dir, branch, err := m.Prepare(t.Context(), "org", "repo", "main", issue, "test", "", "", Fresh) if tc.foreign || tc.dumb { if err == nil { diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go index 64ab909d1..2fa7cc3d6 100644 --- a/internal/worktree/worktree.go +++ b/internal/worktree/worktree.go @@ -1,6 +1,6 @@ // Package worktree performs archied's git operations with go-git: clone, -// branch, commit, push, diff and cleanup. Each task gets an independent full -// clone. +// branch, commit, push, diff and cleanup. Each task gets its own clone, which +// borrows objects from a per-repository mirror. package worktree import ( @@ -14,9 +14,11 @@ import ( "path/filepath" "slices" "strings" + "sync" "time" "github.com/go-git/go-billy/v6" + "github.com/go-git/go-billy/v6/osfs" "github.com/go-git/go-git/v6" gitconfig "github.com/go-git/go-git/v6/config" "github.com/go-git/go-git/v6/plumbing" @@ -50,6 +52,10 @@ type Manager struct { // Log optionally receives diagnostics and non-fatal warnings from // worktree operations. Log *slog.Logger + + // mirrorLocks serialises mirror refreshes per repository directory. + mirrorMu sync.Mutex + mirrorLocks map[string]*sync.Mutex } func (m *Manager) logger() *slog.Logger { @@ -163,11 +169,11 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { return fmt.Errorf("create worktree parent: %w", err) } - r, err := git.PlainCloneContext(ctx, dir, &git.CloneOptions{ - URL: m.cloneURL(owner, repo), - ClientOptions: m.auth(), - ReferenceName: plumbing.NewBranchReferenceName(base), - }) + mirror, err := m.ensureMirror(ctx, owner, repo) + if err != nil { + return err + } + r, err := m.cloneFromMirror(mirror, owner, repo, base, dir) if err != nil { return fmt.Errorf("clone %s/%s: %w", owner, repo, err) } @@ -177,6 +183,168 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string return writeSentinel(dir) } +// mirrorDir is the bare repository whose objects every task clone of +// owner/repo shares. +func (m *Manager) mirrorDir(owner, repo string) string { + return filepath.Join(filepath.Dir(m.Dir(owner, repo, 1)), ".mirror") +} + +func (m *Manager) mirrorLock(dir string) *sync.Mutex { + m.mirrorMu.Lock() + defer m.mirrorMu.Unlock() + if m.mirrorLocks == nil { + m.mirrorLocks = map[string]*sync.Mutex{} + } + if m.mirrorLocks[dir] == nil { + m.mirrorLocks[dir] = &sync.Mutex{} + } + return m.mirrorLocks[dir] +} + +// ensureMirror brings the repository's shared mirror up to date, creating it +// on first use and recreating it when it tracks a different URL. +func (m *Manager) ensureMirror(ctx context.Context, owner, repo string) (string, error) { + dir := m.mirrorDir(owner, repo) + lock := m.mirrorLock(dir) + lock.Lock() + defer lock.Unlock() + + url := m.cloneURL(owner, repo) + if !repositoryUsesURL(dir, url) { + if err := m.createMirror(ctx, dir, url); err != nil { + return "", fmt.Errorf("mirror %s/%s: %w", owner, repo, err) + } + return dir, nil + } + r, err := openRepo(dir) + if err != nil { + return "", fmt.Errorf("open mirror %s/%s: %w", owner, repo, err) + } + if err := r.FetchContext(ctx, &git.FetchOptions{ClientOptions: m.auth(), Tags: plumbing.AllTags}); err != nil && + !errors.Is(err, git.NoErrAlreadyUpToDate) { + return "", fmt.Errorf("fetch mirror %s/%s: %w", owner, repo, err) + } + return dir, nil +} + +// createMirror builds a bare repository tracking every branch of url. It is +// assembled beside dir and renamed into place so a failed fetch leaves nothing +// that looks usable. +func (m *Manager) createMirror(ctx context.Context, dir, url string) error { + tmp := dir + ".tmp" + if err := os.RemoveAll(tmp); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return err + } + r, err := git.PlainInit(tmp, true) + if err != nil { + return err + } + if _, err := r.CreateRemote(&gitconfig.RemoteConfig{ + Name: git.DefaultRemoteName, + URLs: []string{url}, + Fetch: []gitconfig.RefSpec{"+refs/heads/*:refs/heads/*"}, + }); err != nil { + return err + } + if err := r.FetchContext(ctx, &git.FetchOptions{ClientOptions: m.auth(), Tags: plumbing.AllTags}); err != nil { + _ = os.RemoveAll(tmp) + return err + } + if err := os.RemoveAll(dir); err != nil { + return err + } + return os.Rename(tmp, dir) +} + +// cloneFromMirror creates a clone at dir that borrows the mirror's objects via +// alternates and copies its branches in as origin/*, so nothing is transferred. +// Origin is the real forge, so fetch and push behave as for a direct clone. +// (go-git's own Shared clone still downloads a full pack from the mirror.) +func (m *Manager) cloneFromMirror(mirror, owner, repo, base, dir string) (*git.Repository, error) { + r, err := initSharedRepo(mirror, dir) + if err != nil { + return nil, err + } + if _, err := r.CreateRemote(&gitconfig.RemoteConfig{ + Name: git.DefaultRemoteName, + URLs: []string{m.cloneURL(owner, repo)}, + Fetch: []gitconfig.RefSpec{gitconfig.RefSpec(fmt.Sprintf(gitconfig.DefaultFetchRefSpec, git.DefaultRemoteName))}, + }); err != nil { + return nil, err + } + tip, err := importBranches(mirror, r, base) + if err != nil { + return nil, err + } + branch := plumbing.NewBranchReferenceName(base) + if err := r.Storer.SetReference(plumbing.NewHashReference(branch, tip)); err != nil { + return nil, err + } + if err := r.Storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, branch)); err != nil { + return nil, err + } + if err := r.CreateBranch(&gitconfig.Branch{Name: base, Remote: git.DefaultRemoteName, Merge: branch}); err != nil { + return nil, err + } + wt, err := r.Worktree() + if err != nil { + return nil, err + } + if err := wt.Reset(&git.ResetOptions{Mode: git.HardReset, Commit: tip}); err != nil { + return nil, err + } + return r, nil +} + +// initSharedRepo creates an empty repository at dir whose object store falls +// back to the mirror's. +func initSharedRepo(mirror, dir string) (*git.Repository, error) { + r, err := git.PlainInit(dir, false) + if err != nil { + return nil, err + } + alt, ok := r.Storer.(interface{ AddAlternate(string) error }) + if !ok { + return nil, errors.New("object storage does not support alternates") + } + if err := alt.AddAlternate(mirror); err != nil { + return nil, err + } + // Reopen so the object store sees the alternates file. + return openRepo(dir) +} + +// importBranches records each mirror branch as origin/ in r and returns +// the tip of base. +func importBranches(mirror string, r *git.Repository, base string) (plumbing.Hash, error) { + src, err := openRepo(mirror) + if err != nil { + return plumbing.ZeroHash, err + } + refs, err := src.Branches() + if err != nil { + return plumbing.ZeroHash, err + } + defer refs.Close() + var tip plumbing.Hash + if err := refs.ForEach(func(ref *plumbing.Reference) error { + if ref.Name() == plumbing.NewBranchReferenceName(base) { + tip = ref.Hash() + } + return r.Storer.SetReference(plumbing.NewHashReference( + plumbing.NewRemoteReferenceName(git.DefaultRemoteName, ref.Name().Short()), ref.Hash())) + }); err != nil { + return plumbing.ZeroHash, err + } + if tip.IsZero() { + return plumbing.ZeroHash, fmt.Errorf("%w: %s", ErrBaseBranchNotFound, base) + } + return tip, nil +} + // sync positions an already-prepared worktree: Fresh resets it onto // origin/; a resume resets it onto origin/. Both fetch first and // clear the worktree against the target tree before committing to it. @@ -217,7 +385,7 @@ func (m *Manager) migrateLegacy(owner, repo string, issue int, dir string) (bool } func repositoryUsesURL(dir, want string) bool { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return false } @@ -235,7 +403,7 @@ func repositoryUsesURL(dir, want string) bool { // refresh brings an already-prepared worktree back in line with the // remote: fetch, move onto the task branch, and reset it to the base. func (m *Manager) refresh(ctx context.Context, dir, base, branch string) error { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return fmt.Errorf("open prepared worktree: %w", err) } @@ -259,7 +427,7 @@ func (m *Manager) refresh(ctx context.Context, dir, base, branch string) error { // resume resets an existing worktree onto origin/. func (m *Manager) resume(ctx context.Context, dir, branch string) error { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return fmt.Errorf("open prepared worktree: %w", err) } @@ -438,7 +606,7 @@ func (m *Manager) CommitAll(ctx context.Context, dir, message string) (bool, err if err := ctx.Err(); err != nil { return false, err } - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return false, fmt.Errorf("open worktree: %w", err) } @@ -487,7 +655,7 @@ func (m *Manager) PushRepository(ctx context.Context, owner, repo string, issue } func (m *Manager) push(ctx context.Context, dir, branch, remoteURL string) error { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return fmt.Errorf("open worktree: %w", err) } @@ -572,7 +740,7 @@ func (m *Manager) patch(ctx context.Context, dir, base string) (*object.Patch, e // patchFromMergeBase returns the patch and the merge-base commit it was taken // against. func (m *Manager) patchFromMergeBase(ctx context.Context, dir, base string) (*object.Patch, plumbing.Hash, error) { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return nil, plumbing.ZeroHash, fmt.Errorf("open worktree: %w", err) } @@ -612,7 +780,7 @@ func (m *Manager) HasUncommittedChanges(ctx context.Context, dir string) (bool, if err := ctx.Err(); err != nil { return false, err } - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return false, fmt.Errorf("open worktree: %w", err) } @@ -686,7 +854,7 @@ func (m *Manager) ChangedFileStats(ctx context.Context, dir, base string) (task. if err != nil { return task.ChangeStats{}, err } - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return task.ChangeStats{}, fmt.Errorf("open worktree: %w", err) } @@ -759,7 +927,7 @@ func fileChange(fp fdiff.FilePatch) task.FileChange { // Snapshot exports HEAD's tracked files into destDir, without .git. func (m *Manager) Snapshot(ctx context.Context, dir, destDir string) error { - r, err := git.PlainOpen(dir) + r, err := openRepo(dir) if err != nil { return fmt.Errorf("open worktree: %w", err) } @@ -1016,3 +1184,9 @@ func (m *Manager) CheckAccess(ctx context.Context, owner, repo, base string) err } return fmt.Errorf("%w: %s", ErrBaseBranchNotFound, base) } + +// openRepo opens a repository whose objects may live in the mirror it borrows +// from, which sits outside the repository's own directory. +func openRepo(dir string) (*git.Repository, error) { + return git.PlainOpenWithOptions(dir, &git.PlainOpenOptions{AlternatesFS: osfs.New(string(filepath.Separator))}) +}