fix(logging): keep the live descriptor open across a failed rotation

rotate() closed the current file before attempting the rename+reopen, so
when the reopen failed (a full disk, a permission change) every write
afterward went to a closed *os.File and was silently dropped -- the sink
stayed permanently broken rather than recovering once the transient
condition cleared. Keep the old descriptor open across the rename/reopen
and only close it once the new one is confirmed, so a write during (or
after) a failed rotation always lands somewhere.
This commit is contained in:
sam committed 2026-09-15 04:55:48 +10:00
1 parent 6f26d790ba
commit bb4e2f9135
2 files changed
+97 -7

No files matched your search

+23 -7
View File
@@ -125,8 +125,13 @@ func newRotatingFile(path string, maxSizeMB, keep int) (*rotatingFile, error) {
return r, nil
}
// openFile is os.OpenFile by default; tests override it to inject a
// deterministic reopen failure without needing filesystem-permission
// tricks that a failed rename would also trip.
var openFile = os.OpenFile
func (r *rotatingFile) open() error {
f, err := os.OpenFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
f, err := openFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
if err != nil {
return err
}
@@ -153,18 +158,29 @@ func (r *rotatingFile) Write(p []byte) (int, error) {
}
// rotate renames the live file to .1, shifting existing generations down and
// discarding anything past keep.
// discarding anything past keep, then opens a fresh file at path.
//
// The live descriptor is kept open across the renames -- POSIX rename only
// changes a directory entry, not what an open file descriptor points at --
// so if the rename or the reopen below fails, r.f still refers to a
// perfectly writable file (the original, or the just-rotated one) and the
// caller's write is never lost. Closing the old descriptor only happens
// once the new one is confirmed open.
func (r *rotatingFile) rotate() error {
if err := r.f.Close(); err != nil {
return err
}
// Oldest first, so each rename lands on a free slot.
_ = os.Remove(fmt.Sprintf("%s.%d", r.path, r.keep))
for i := r.keep - 1; i >= 1; i-- {
_ = os.Rename(fmt.Sprintf("%s.%d", r.path, i), fmt.Sprintf("%s.%d", r.path, i+1))
}
_ = os.Rename(r.path, r.path+".1")
return r.open()
if err := os.Rename(r.path, r.path+".1"); err != nil {
return err
}
old := r.f
if err := r.open(); err != nil {
r.f = old
return err
}
return old.Close()
}
func (r *rotatingFile) Close() error {
+74
View File
@@ -156,6 +156,80 @@ func TestRotation(t *testing.T) {
}
}
// TestRotationSurvivesFailedReopen pins that a failed rotation reopen (a
// full filesystem, a permission change hit exactly when archied tries to
// recreate the live file) does not take down the file sink permanently.
// The write that triggered rotation must not be silently swallowed, and
// -- unlike losing a single line -- every write after the failure must
// not be lost either.
func TestRotationSurvivesFailedReopen(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "archied.log")
w, err := newRotatingFile(path, 1, 2)
if err != nil {
t.Fatalf("newRotatingFile: %v", err)
}
// Push size right up to the cap without writing megabytes of
// fixture data -- white-box, same package -- so the marker write
// below is guaranteed to be the one that crosses maxSize and
// triggers rotate.
marker := []byte("MARKER-LINE\n")
w.size = w.maxSize - 1
// Simulate the reopen failing (disk full, permission change) while
// the rename that frees the old name still succeeds -- the scenario
// a chmod on the directory can't isolate, since that would also
// block the rename.
prevOpenFile := openFile
openFile = func(name string, flag int, perm os.FileMode) (*os.File, error) {
return nil, fmt.Errorf("simulated reopen failure")
}
t.Cleanup(func() { openFile = prevOpenFile })
if _, err := w.Write(marker); err != nil {
t.Fatalf("Write after a failed reopen returned an error (line lost): %v", err)
}
// A second write after the same failure must also land -- this is
// what distinguishes "lost one line during rotation" from "the sink
// is now permanently broken".
marker2 := []byte("MARKER-LINE-2\n")
if _, err := w.Write(marker2); err != nil {
t.Fatalf("second write after a failed reopen returned an error: %v", err)
}
openFile = prevOpenFile
if err := w.Close(); err != nil {
t.Fatal(err)
}
// The live file was never successfully recreated (reopen kept
// failing on every write), so each write's rotate attempt renamed
// the same still-open descriptor further down the generation chain
// -- the exact filename it lands under is incidental. What matters
// is that neither marker is gone: scan the whole directory.
var all strings.Builder
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
for _, e := range entries {
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
if err != nil {
t.Fatal(err)
}
all.Write(b)
}
if !strings.Contains(all.String(), "MARKER-LINE\n") {
t.Error("MARKER-LINE missing after a failed reopen")
}
if !strings.Contains(all.String(), "MARKER-LINE-2\n") {
t.Error("MARKER-LINE-2 missing -- the sink stayed broken after the transient failure")
}
}
func TestRotationDefaults(t *testing.T) {
path := filepath.Join(t.TempDir(), "archied.log")
w, err := newRotatingFile(path, 0, 0)