fix(worktree): recover retries from path-type conflicts and stop masking the cause

A retry after any attempt that did not commit failed with

  worktree prepare failed: checkout branch <branch>:
  a branch named "refs/heads/<branch>" already exists

on a branch that was never the problem. refresh inferred "the branch does
not exist yet" from a failed checkout and reported whatever the Create
fallback then returned, so the real failure was replaced by a false one --
and the operator had no way to tell that the park reason was fiction.

The real failure, reproduced: a stage running in the container can leave a
path whose TYPE contradicts the tree refresh resets onto -- a directory
where the base tree has a file, or a file where the base tree has a
directory. go-git's hard reset cannot write over that contradiction and
fails with `openat <path>: is a directory` / `not a directory`, which is
exactly the failure the Create fallback then relabelled. Retry could not
recover from it, and a second path of the same class (a branch removed
while HEAD still pointed at it) failed with "reference not found".

refresh now clears the worktree against the tree it is resetting ONTO
before it checks anything out, which removes exactly those conflicts -- an
on-disk directory the tree declares as a file is untracked, and so is a
file the tree declares as a directory -- and resolves the branch's
existence from the reference store instead of inferring it. Resume gets the
same treatment, so the remediate path cannot fail the same way.

This is the third fix to this function (Force in 05944b6f was the second),
so the checkout/reset/clean sequence is rewritten rather than patched again.
This commit is contained in:
sam committed 2026-09-18 01:49:28 +10:00
1 parent 583187c968
commit fe3d508b6c
2 files changed
+276 -52

No files matched your search

+93 -51
View File
@@ -242,40 +242,15 @@ func (m *Manager) refresh(ctx context.Context, dir, base, branch string) error {
return fmt.Errorf("fetch origin: %w", err)
}
wt, err := r.Worktree()
if err != nil {
return fmt.Errorf("open worktree: %w", err)
}
// Force discards whatever the working tree looks like right now: an
// interrupted prior attempt (killed container, a stage that edits
// files before ever reaching commit) can leave local modifications
// that make go-git's default MergeReset refuse to reconcile them, even
// onto a commit HEAD is already on. That refusal used to surface as
// "branch already exists" from the fallback below, which mis-assumed
// any checkout failure meant the branch didn't exist yet. Force is
// safe here regardless of cause: the hard reset to base a few lines
// down is about to discard the working tree anyway.
ref := plumbing.NewBranchReferenceName(branch)
if err := wt.Checkout(&git.CheckoutOptions{Branch: ref, Force: true}); err != nil {
// The branch does not exist yet in this worktree.
if err := wt.Checkout(&git.CheckoutOptions{Branch: ref, Create: true, Force: true}); err != nil {
return fmt.Errorf("checkout branch %s: %w", branch, err)
}
}
// Reset onto the freshly fetched base. This is a prepared clone, so a
// missing base is configuration or repository drift and must fail closed.
// missing base is configuration or repository drift and must fail closed
// -- and it is resolved before the first mutation, so a failure here
// leaves the worktree as it was rather than half-refreshed.
baseHash, err := resolveBase(r, base)
if err != nil {
return err
}
if err := wt.Reset(&git.ResetOptions{Commit: baseHash, Mode: git.HardReset}); err != nil {
return fmt.Errorf("reset to %s: %w", remoteBase(base), err)
}
if err := cleanUntracked(r, dir); err != nil {
return fmt.Errorf("clean abandoned worktree files: %w", err)
}
return nil
return resetOnto(r, dir, branch, baseHash, remoteBase(base))
}
// Resume re-syncs an already-prepared worktree onto its branch's remote tip
@@ -295,41 +270,104 @@ func (m *Manager) Resume(ctx context.Context, dir, branch string) error {
return fmt.Errorf("fetch origin: %w", err)
}
wt, err := r.Worktree()
if err != nil {
return fmt.Errorf("open worktree: %w", err)
}
ref := plumbing.NewBranchReferenceName(branch)
if err := wt.Checkout(&git.CheckoutOptions{Branch: ref, Force: true}); err != nil {
return fmt.Errorf("checkout branch %s: %w", branch, err)
}
tip, err := r.ResolveRevision(plumbing.Revision(remoteBase(branch)))
if err != nil {
return fmt.Errorf("resolve remote tip of %s: %w", branch, err)
}
if err := wt.Reset(&git.ResetOptions{Commit: *tip, Mode: git.HardReset}); err != nil {
return fmt.Errorf("reset to %s: %w", remoteBase(branch), err)
return resetOnto(r, dir, branch, *tip, remoteBase(branch))
}
// resetOnto discards the worktree's current contents and points branch at
// commit, leaving a clean tree that matches commit.
//
// The ORDER here is the substance of the function, and it is deliberately the
// opposite of the checkout-then-reset-then-clean it replaced. A previous
// attempt -- an interrupted stage, a killed container, or a build run inside
// the root sandbox -- can leave a path whose type contradicts the tree being
// reset onto as well as untracked output beside it. go-git's reset cannot
// write a file where a directory sits, nor create a directory through a file;
// it failed with `openat <path>: is a directory` / `not a directory` before it
// could apply anything, and the Create-fallback that used to follow reported
// that as a branch that already existed. Clearing the worktree against the
// TARGET tree first removes exactly those conflicts -- an on-disk directory the
// tree declares as a file is untracked, and so is a file the tree declares as a
// directory -- so the reset that follows has nothing left to collide with.
func resetOnto(r *git.Repository, dir, branch string, commit plumbing.Hash, label string) error {
target, err := commitTree(r, commit)
if err != nil {
return err
}
if err := cleanUntracked(r, dir); err != nil {
return fmt.Errorf("clean abandoned worktree files: %w", err)
if err := cleanUntracked(dir, target); err != nil {
return fmt.Errorf("clear abandoned worktree files: %w", err)
}
wt, err := r.Worktree()
if err != nil {
return fmt.Errorf("open worktree: %w", err)
}
if err := checkOutBranch(r, wt, branch, commit); err != nil {
return err
}
if err := wt.Reset(&git.ResetOptions{Commit: commit, Mode: git.HardReset}); err != nil {
return fmt.Errorf("reset to %s: %w", label, err)
}
return nil
}
func cleanUntracked(r *git.Repository, dir string) error {
head, err := r.Head()
if err != nil {
return fmt.Errorf("resolve HEAD: %w", err)
// checkOutBranch switches the worktree to branch, creating it at commit when it
// does not exist yet.
//
// Whether the branch exists is resolved from the reference store rather than
// inferred from a failed checkout. A checkout fails for reasons that have
// nothing to do with a missing branch -- most often a path in the working tree
// the tree cannot be written over -- and treating any such failure as "the
// branch does not exist yet" produced `a branch named "refs/heads/<branch>"
// already exists`: it named a branch that was never the problem and hid the
// real cause from the park reason an operator reads to decide what to do.
//
// Creating from an explicit commit rather than from HEAD matters too: a branch
// removed while HEAD still pointed at it (a renamed branch, a half-finished
// refresh) made go-git's create path read an unresolvable HEAD and fail with
// "reference not found" instead of creating the branch.
func checkOutBranch(r *git.Repository, wt *git.Worktree, branch string, commit plumbing.Hash) error {
ref := plumbing.NewBranchReferenceName(branch)
opts := &git.CheckoutOptions{Branch: ref, Force: true}
switch _, err := r.Reference(ref, false); {
case err == nil:
case errors.Is(err, plumbing.ErrReferenceNotFound):
opts.Create = true
opts.Hash = commit
default:
return fmt.Errorf("resolve branch %s: %w", branch, err)
}
commit, err := r.CommitObject(head.Hash())
if err != nil {
return fmt.Errorf("load HEAD commit: %w", err)
if err := wt.Checkout(opts); err != nil {
return fmt.Errorf("checkout branch %s: %w", branch, err)
}
tree, err := commit.Tree()
return nil
}
// commitTree loads the tree of one commit, for the caller that has to know
// which paths the worktree is about to be reset onto.
func commitTree(r *git.Repository, commit plumbing.Hash) (*object.Tree, error) {
c, err := r.CommitObject(commit)
if err != nil {
return fmt.Errorf("load HEAD tree: %w", err)
return nil, fmt.Errorf("load commit %s: %w", commit, err)
}
tree, err := c.Tree()
if err != nil {
return nil, fmt.Errorf("load tree of %s: %w", commit, err)
}
return tree, nil
}
// cleanUntracked removes every path under dir that tree does not contain:
// untracked files, whole directories of build output, and any path whose type
// on disk contradicts what the tree declares.
//
// It is given the tree the worktree is being reset ONTO, not HEAD: the point is
// to leave nothing behind that the incoming tree cannot be written over, and
// on a retry HEAD is one of the things being replaced.
func cleanUntracked(dir string, tree *object.Tree) error {
files := make(map[string]struct{})
dirs := make(map[string]struct{})
opaqueDirs := make(map[string]struct{})
@@ -367,6 +405,10 @@ func cleanUntracked(r *git.Repository, dir string) error {
if _, ok := dirs[rel]; ok {
return nil
}
// A directory the tree does not declare as one -- including a
// directory sitting on a path the tree declares as a FILE -- is
// output, not tracked content. It has to go before the reset can
// write the file this path is supposed to hold.
if err := os.RemoveAll(path); err != nil {
return err
}
+183 -1
View File
@@ -324,6 +324,176 @@ func TestPrepareRecoversFromADirtyWorktreeOnRetry(t *testing.T) {
}
}
// TestPrepareRecoversFromAPathTypeConflictOnRetry covers the retry failure
// that followed the dirty-worktree one above, and the first that was not
// about the *contents* of a file. A stage running inside the container can
// leave a path whose TYPE contradicts the tree refresh resets onto: a
// directory where the base tree has a file, or a file where the base tree
// has a directory. go-git's hard reset cannot write over that contradiction --
// it fails with `openat <path>: is a directory` / `not a directory` -- and the
// Create-fallback in refresh then reported the whole thing as
// `checkout branch <branch>: a branch named "refs/heads/<branch>" already
// exists`. That names a branch that was never the problem, and it is the
// reason a retry could not recover: the reported obstacle was not the real
// one.
//
// refresh's contract is to discard the working tree, so a path it cannot
// reset over is a path it has to clear first.
func TestPrepareRecoversFromAPathTypeConflictOnRetry(t *testing.T) {
cases := []struct {
name string
wreck func(t *testing.T, dir string)
}{
{"directory where the tree has a file", func(t *testing.T, dir string) {
if err := os.Remove(filepath.Join(dir, "README.md")); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "README.md", "inner"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "README.md", "inner", "junk"), []byte("junk\n"), 0o600); err != nil {
t.Fatal(err)
}
}},
{"file where the tree has a directory", func(t *testing.T, dir string) {
if err := os.RemoveAll(filepath.Join(dir, "pkg")); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "pkg"), []byte("now a file\n"), 0o600); err != nil {
t.Fatal(err)
}
}},
{"directory where the tree has a nested file", func(t *testing.T, dir string) {
if err := os.RemoveAll(filepath.Join(dir, "pkg")); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(filepath.Join(dir, "pkg", "a.txt"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "pkg", "a.txt", "junk"), []byte("junk\n"), 0o600); err != nil {
t.Fatal(err)
}
}},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
ctx := context.Background()
host := newLocalRemote(t, "acme", "todo")
seedTrackedPath(t, host, "pkg/a.txt", "a\n")
m := newManager(t, host)
dir, branch, err := m.Prepare(ctx, "acme", "todo", testBase, 21, "fix: path type conflict", "", "bug")
if err != nil {
t.Fatalf("first Prepare() error = %v", err)
}
c.wreck(t, dir)
dir2, branch2, err := m.Prepare(ctx, "acme", "todo", testBase, 21, "fix: path type conflict", "", "bug")
if err != nil {
t.Fatalf("retry Prepare() over a path type conflict error = %v, want recovery", err)
}
if dir2 != dir || branch2 != branch {
t.Errorf("retry Prepare() = (%q, %q), want the same (%q, %q)", dir2, branch2, dir, branch)
}
// The retry must leave the base tree's own shape behind, not the
// wreckage: whichever type the previous attempt left, the tracked
// path has to come back as the tree declares it.
info, err := os.Lstat(filepath.Join(dir, "README.md"))
if err != nil {
t.Fatalf("README.md after retry: %v", err)
}
if info.IsDir() {
t.Error("README.md is still a directory, want the base tree's regular file")
}
content, err := os.ReadFile(filepath.Join(dir, "README.md"))
if err != nil {
t.Fatalf("read README.md after retry: %v", err)
}
if string(content) != "seed\n" {
t.Errorf("README.md = %q, want the base commit's content restored", content)
}
if pkg, err := os.Stat(filepath.Join(dir, "pkg")); err != nil {
t.Fatalf("pkg after retry: %v", err)
} else if !pkg.IsDir() {
t.Error("pkg is still a file, want the base tree's directory")
}
if a, err := os.ReadFile(filepath.Join(dir, "pkg", "a.txt")); err != nil {
t.Fatalf("read pkg/a.txt after retry: %v", err)
} else if string(a) != "a\n" {
t.Errorf("pkg/a.txt = %q, want the base commit's content restored", a)
}
})
}
}
// TestPrepareCreatesTheBranchWhenItIsGenuinelyMissing keeps the other half of
// refresh's branch handling honest: resolving a branch's existence before
// deciding whether to create it must not lose the create path. The branch name
// derives from the issue title, so an edited title between attempts renames the
// branch, and the second attempt has to create it.
func TestPrepareCreatesTheBranchWhenItIsGenuinelyMissing(t *testing.T) {
ctx := context.Background()
host := newLocalRemote(t, "acme", "todo")
m := newManager(t, host)
dir, _, err := m.Prepare(ctx, "acme", "todo", testBase, 22, "fix: first title", "", "bug")
if err != nil {
t.Fatalf("first Prepare() error = %v", err)
}
r, err := git.PlainOpen(dir)
if err != nil {
t.Fatal(err)
}
first := archieBranch(22, "fix: first title", "bug")
if err := r.Storer.RemoveReference(plumbing.NewBranchReferenceName(first)); err != nil {
t.Fatalf("remove branch %s: %v", first, err)
}
_, branch, err := m.Prepare(ctx, "acme", "todo", testBase, 22, "fix: renamed title", "", "bug")
if err != nil {
t.Fatalf("Prepare() after the branch was removed error = %v, want it created", err)
}
if _, err := r.Reference(plumbing.NewBranchReferenceName(branch), false); err != nil {
t.Errorf("branch %q not present after Prepare(): %v", branch, err)
}
}
// seedTrackedPath commits one path (creating its parent directories) onto the
// fake remote's base branch, so a fixture can have a tracked *directory* as
// well as the single tracked file newLocalRemote seeds.
func seedTrackedPath(t *testing.T, host, path, content string) {
t.Helper()
seedDir := filepath.Join(t.TempDir(), "path-seed")
r, err := git.PlainClone(seedDir, &git.CloneOptions{URL: filepath.Join(host, "acme", "todo.git")})
if err != nil {
t.Fatalf("clone path seed: %v", err)
}
full := filepath.Join(seedDir, filepath.FromSlash(path))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
disableSigning(t, r)
wt, err := r.Worktree()
if err != nil {
t.Fatal(err)
}
if _, err := wt.Add(path); err != nil {
t.Fatalf("add %s: %v", path, err)
}
sig := testSignature()
if _, err := wt.Commit("chore: seed "+path, &git.CommitOptions{Author: sig, Committer: sig}); err != nil {
t.Fatalf("commit %s: %v", path, err)
}
if err := r.Push(&git.PushOptions{RemoteName: git.DefaultRemoteName}); err != nil {
t.Fatalf("push %s: %v", path, err)
}
}
// A repository whose .git is a gitdir *file* rather than a directory (a
// linked worktree, or a clone adopted by migrateLegacy) must still have its
// abandoned untracked files cleaned. Returning filepath.SkipDir for a
@@ -359,7 +529,19 @@ func TestCleanUntrackedHandlesGitdirFile(t *testing.T) {
if err != nil {
t.Fatalf("PlainOpen() with a gitdir file error = %v", err)
}
if err := cleanUntracked(r, dir); err != nil {
head, err := r.Head()
if err != nil {
t.Fatal(err)
}
headCommit, err := r.CommitObject(head.Hash())
if err != nil {
t.Fatal(err)
}
headTree, err := headCommit.Tree()
if err != nil {
t.Fatal(err)
}
if err := cleanUntracked(dir, headTree); err != nil {
t.Fatalf("cleanUntracked() error = %v", err)
}
if _, err := os.Stat(abandoned); !os.IsNotExist(err) {