perf(worktree): clone tasks from a shared per-repository mirror
This commit is contained in:
1 parent
e6d765d233
commit
74e34fb07a
2 files changed
+194
-19
No files matched your search
@@ -108,9 +108,10 @@ func TestForgeHTTPAuthentication(t *testing.T) {
|
||||
backend.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
m := *manager
|
||||
m.BaseURL = server.URL
|
||||
m.WorkDir = t.TempDir()
|
||||
m := &Manager{
|
||||
BotUser: manager.BotUser, BotEmail: manager.BotEmail, Token: manager.Token,
|
||||
BaseURL: server.URL, WorkDir: t.TempDir(),
|
||||
}
|
||||
dir, branch, err := m.Prepare(t.Context(), "org", "repo", "main", issue, "test", "", "", Fresh)
|
||||
if tc.foreign || tc.dumb {
|
||||
if err == nil {
|
||||
|
||||
+190
-16
@@ -1,6 +1,6 @@
|
||||
// Package worktree performs archied's git operations with go-git: clone,
|
||||
// branch, commit, push, diff and cleanup. Each task gets an independent full
|
||||
// clone.
|
||||
// branch, commit, push, diff and cleanup. Each task gets its own clone, which
|
||||
// borrows objects from a per-repository mirror.
|
||||
package worktree
|
||||
|
||||
import (
|
||||
@@ -14,9 +14,11 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-billy/v6"
|
||||
"github.com/go-git/go-billy/v6/osfs"
|
||||
"github.com/go-git/go-git/v6"
|
||||
gitconfig "github.com/go-git/go-git/v6/config"
|
||||
"github.com/go-git/go-git/v6/plumbing"
|
||||
@@ -50,6 +52,10 @@ type Manager struct {
|
||||
// Log optionally receives diagnostics and non-fatal warnings from
|
||||
// worktree operations.
|
||||
Log *slog.Logger
|
||||
|
||||
// mirrorLocks serialises mirror refreshes per repository directory.
|
||||
mirrorMu sync.Mutex
|
||||
mirrorLocks map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
func (m *Manager) logger() *slog.Logger {
|
||||
@@ -163,11 +169,11 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string
|
||||
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||
return fmt.Errorf("create worktree parent: %w", err)
|
||||
}
|
||||
r, err := git.PlainCloneContext(ctx, dir, &git.CloneOptions{
|
||||
URL: m.cloneURL(owner, repo),
|
||||
ClientOptions: m.auth(),
|
||||
ReferenceName: plumbing.NewBranchReferenceName(base),
|
||||
})
|
||||
mirror, err := m.ensureMirror(ctx, owner, repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := m.cloneFromMirror(mirror, owner, repo, base, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("clone %s/%s: %w", owner, repo, err)
|
||||
}
|
||||
@@ -177,6 +183,168 @@ func (m *Manager) createClone(ctx context.Context, owner, repo, base, dir string
|
||||
return writeSentinel(dir)
|
||||
}
|
||||
|
||||
// mirrorDir is the bare repository whose objects every task clone of
|
||||
// owner/repo shares.
|
||||
func (m *Manager) mirrorDir(owner, repo string) string {
|
||||
return filepath.Join(filepath.Dir(m.Dir(owner, repo, 1)), ".mirror")
|
||||
}
|
||||
|
||||
func (m *Manager) mirrorLock(dir string) *sync.Mutex {
|
||||
m.mirrorMu.Lock()
|
||||
defer m.mirrorMu.Unlock()
|
||||
if m.mirrorLocks == nil {
|
||||
m.mirrorLocks = map[string]*sync.Mutex{}
|
||||
}
|
||||
if m.mirrorLocks[dir] == nil {
|
||||
m.mirrorLocks[dir] = &sync.Mutex{}
|
||||
}
|
||||
return m.mirrorLocks[dir]
|
||||
}
|
||||
|
||||
// ensureMirror brings the repository's shared mirror up to date, creating it
|
||||
// on first use and recreating it when it tracks a different URL.
|
||||
func (m *Manager) ensureMirror(ctx context.Context, owner, repo string) (string, error) {
|
||||
dir := m.mirrorDir(owner, repo)
|
||||
lock := m.mirrorLock(dir)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
url := m.cloneURL(owner, repo)
|
||||
if !repositoryUsesURL(dir, url) {
|
||||
if err := m.createMirror(ctx, dir, url); err != nil {
|
||||
return "", fmt.Errorf("mirror %s/%s: %w", owner, repo, err)
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("open mirror %s/%s: %w", owner, repo, err)
|
||||
}
|
||||
if err := r.FetchContext(ctx, &git.FetchOptions{ClientOptions: m.auth(), Tags: plumbing.AllTags}); err != nil &&
|
||||
!errors.Is(err, git.NoErrAlreadyUpToDate) {
|
||||
return "", fmt.Errorf("fetch mirror %s/%s: %w", owner, repo, err)
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// createMirror builds a bare repository tracking every branch of url. It is
|
||||
// assembled beside dir and renamed into place so a failed fetch leaves nothing
|
||||
// that looks usable.
|
||||
func (m *Manager) createMirror(ctx context.Context, dir, url string) error {
|
||||
tmp := dir + ".tmp"
|
||||
if err := os.RemoveAll(tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
r, err := git.PlainInit(tmp, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := r.CreateRemote(&gitconfig.RemoteConfig{
|
||||
Name: git.DefaultRemoteName,
|
||||
URLs: []string{url},
|
||||
Fetch: []gitconfig.RefSpec{"+refs/heads/*:refs/heads/*"},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.FetchContext(ctx, &git.FetchOptions{ClientOptions: m.auth(), Tags: plumbing.AllTags}); err != nil {
|
||||
_ = os.RemoveAll(tmp)
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, dir)
|
||||
}
|
||||
|
||||
// cloneFromMirror creates a clone at dir that borrows the mirror's objects via
|
||||
// alternates and copies its branches in as origin/*, so nothing is transferred.
|
||||
// Origin is the real forge, so fetch and push behave as for a direct clone.
|
||||
// (go-git's own Shared clone still downloads a full pack from the mirror.)
|
||||
func (m *Manager) cloneFromMirror(mirror, owner, repo, base, dir string) (*git.Repository, error) {
|
||||
r, err := initSharedRepo(mirror, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := r.CreateRemote(&gitconfig.RemoteConfig{
|
||||
Name: git.DefaultRemoteName,
|
||||
URLs: []string{m.cloneURL(owner, repo)},
|
||||
Fetch: []gitconfig.RefSpec{gitconfig.RefSpec(fmt.Sprintf(gitconfig.DefaultFetchRefSpec, git.DefaultRemoteName))},
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tip, err := importBranches(mirror, r, base)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
branch := plumbing.NewBranchReferenceName(base)
|
||||
if err := r.Storer.SetReference(plumbing.NewHashReference(branch, tip)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.Storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, branch)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.CreateBranch(&gitconfig.Branch{Name: base, Remote: git.DefaultRemoteName, Merge: branch}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wt, err := r.Worktree()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := wt.Reset(&git.ResetOptions{Mode: git.HardReset, Commit: tip}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// initSharedRepo creates an empty repository at dir whose object store falls
|
||||
// back to the mirror's.
|
||||
func initSharedRepo(mirror, dir string) (*git.Repository, error) {
|
||||
r, err := git.PlainInit(dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
alt, ok := r.Storer.(interface{ AddAlternate(string) error })
|
||||
if !ok {
|
||||
return nil, errors.New("object storage does not support alternates")
|
||||
}
|
||||
if err := alt.AddAlternate(mirror); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reopen so the object store sees the alternates file.
|
||||
return openRepo(dir)
|
||||
}
|
||||
|
||||
// importBranches records each mirror branch as origin/<name> in r and returns
|
||||
// the tip of base.
|
||||
func importBranches(mirror string, r *git.Repository, base string) (plumbing.Hash, error) {
|
||||
src, err := openRepo(mirror)
|
||||
if err != nil {
|
||||
return plumbing.ZeroHash, err
|
||||
}
|
||||
refs, err := src.Branches()
|
||||
if err != nil {
|
||||
return plumbing.ZeroHash, err
|
||||
}
|
||||
defer refs.Close()
|
||||
var tip plumbing.Hash
|
||||
if err := refs.ForEach(func(ref *plumbing.Reference) error {
|
||||
if ref.Name() == plumbing.NewBranchReferenceName(base) {
|
||||
tip = ref.Hash()
|
||||
}
|
||||
return r.Storer.SetReference(plumbing.NewHashReference(
|
||||
plumbing.NewRemoteReferenceName(git.DefaultRemoteName, ref.Name().Short()), ref.Hash()))
|
||||
}); err != nil {
|
||||
return plumbing.ZeroHash, err
|
||||
}
|
||||
if tip.IsZero() {
|
||||
return plumbing.ZeroHash, fmt.Errorf("%w: %s", ErrBaseBranchNotFound, base)
|
||||
}
|
||||
return tip, nil
|
||||
}
|
||||
|
||||
// sync positions an already-prepared worktree: Fresh resets it onto
|
||||
// origin/<base>; a resume resets it onto origin/<branch>. Both fetch first and
|
||||
// clear the worktree against the target tree before committing to it.
|
||||
@@ -217,7 +385,7 @@ func (m *Manager) migrateLegacy(owner, repo string, issue int, dir string) (bool
|
||||
}
|
||||
|
||||
func repositoryUsesURL(dir, want string) bool {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
@@ -235,7 +403,7 @@ func repositoryUsesURL(dir, want string) bool {
|
||||
// refresh brings an already-prepared worktree back in line with the
|
||||
// remote: fetch, move onto the task branch, and reset it to the base.
|
||||
func (m *Manager) refresh(ctx context.Context, dir, base, branch string) error {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open prepared worktree: %w", err)
|
||||
}
|
||||
@@ -259,7 +427,7 @@ func (m *Manager) refresh(ctx context.Context, dir, base, branch string) error {
|
||||
|
||||
// resume resets an existing worktree onto origin/<branch>.
|
||||
func (m *Manager) resume(ctx context.Context, dir, branch string) error {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open prepared worktree: %w", err)
|
||||
}
|
||||
@@ -438,7 +606,7 @@ func (m *Manager) CommitAll(ctx context.Context, dir, message string) (bool, err
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -487,7 +655,7 @@ func (m *Manager) PushRepository(ctx context.Context, owner, repo string, issue
|
||||
}
|
||||
|
||||
func (m *Manager) push(ctx context.Context, dir, branch, remoteURL string) error {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -572,7 +740,7 @@ func (m *Manager) patch(ctx context.Context, dir, base string) (*object.Patch, e
|
||||
// patchFromMergeBase returns the patch and the merge-base commit it was taken
|
||||
// against.
|
||||
func (m *Manager) patchFromMergeBase(ctx context.Context, dir, base string) (*object.Patch, plumbing.Hash, error) {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return nil, plumbing.ZeroHash, fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -612,7 +780,7 @@ func (m *Manager) HasUncommittedChanges(ctx context.Context, dir string) (bool,
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -686,7 +854,7 @@ func (m *Manager) ChangedFileStats(ctx context.Context, dir, base string) (task.
|
||||
if err != nil {
|
||||
return task.ChangeStats{}, err
|
||||
}
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return task.ChangeStats{}, fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -759,7 +927,7 @@ func fileChange(fp fdiff.FilePatch) task.FileChange {
|
||||
|
||||
// Snapshot exports HEAD's tracked files into destDir, without .git.
|
||||
func (m *Manager) Snapshot(ctx context.Context, dir, destDir string) error {
|
||||
r, err := git.PlainOpen(dir)
|
||||
r, err := openRepo(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open worktree: %w", err)
|
||||
}
|
||||
@@ -1016,3 +1184,9 @@ func (m *Manager) CheckAccess(ctx context.Context, owner, repo, base string) err
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrBaseBranchNotFound, base)
|
||||
}
|
||||
|
||||
// openRepo opens a repository whose objects may live in the mirror it borrows
|
||||
// from, which sits outside the repository's own directory.
|
||||
func openRepo(dir string) (*git.Repository, error) {
|
||||
return git.PlainOpenWithOptions(dir, &git.PlainOpenOptions{AlternatesFS: osfs.New(string(filepath.Separator))})
|
||||
}
|
||||
Reference in new issue
Block a user