fix(logging): keep the live descriptor open across a failed rotation
rotate() closed the current file before attempting the rename+reopen, so when the reopen failed (a full disk, a permission change) every write afterward went to a closed *os.File and was silently dropped -- the sink stayed permanently broken rather than recovering once the transient condition cleared. Keep the old descriptor open across the rename/reopen and only close it once the new one is confirmed, so a write during (or after) a failed rotation always lands somewhere.
This commit is contained in:
2 files changed
+97
-7
No files matched your search
@@ -125,8 +125,13 @@ func newRotatingFile(path string, maxSizeMB, keep int) (*rotatingFile, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// openFile is os.OpenFile by default; tests override it to inject a
|
||||
// deterministic reopen failure without needing filesystem-permission
|
||||
// tricks that a failed rename would also trip.
|
||||
var openFile = os.OpenFile
|
||||
|
||||
func (r *rotatingFile) open() error {
|
||||
f, err := os.OpenFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
|
||||
f, err := openFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -153,18 +158,29 @@ func (r *rotatingFile) Write(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
// rotate renames the live file to .1, shifting existing generations down and
|
||||
// discarding anything past keep.
|
||||
// discarding anything past keep, then opens a fresh file at path.
|
||||
//
|
||||
// The live descriptor is kept open across the renames -- POSIX rename only
|
||||
// changes a directory entry, not what an open file descriptor points at --
|
||||
// so if the rename or the reopen below fails, r.f still refers to a
|
||||
// perfectly writable file (the original, or the just-rotated one) and the
|
||||
// caller's write is never lost. Closing the old descriptor only happens
|
||||
// once the new one is confirmed open.
|
||||
func (r *rotatingFile) rotate() error {
|
||||
if err := r.f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Oldest first, so each rename lands on a free slot.
|
||||
_ = os.Remove(fmt.Sprintf("%s.%d", r.path, r.keep))
|
||||
for i := r.keep - 1; i >= 1; i-- {
|
||||
_ = os.Rename(fmt.Sprintf("%s.%d", r.path, i), fmt.Sprintf("%s.%d", r.path, i+1))
|
||||
}
|
||||
_ = os.Rename(r.path, r.path+".1")
|
||||
return r.open()
|
||||
if err := os.Rename(r.path, r.path+".1"); err != nil {
|
||||
return err
|
||||
}
|
||||
old := r.f
|
||||
if err := r.open(); err != nil {
|
||||
r.f = old
|
||||
return err
|
||||
}
|
||||
return old.Close()
|
||||
}
|
||||
|
||||
func (r *rotatingFile) Close() error {
|
||||
|
||||
@@ -156,6 +156,80 @@ func TestRotation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRotationSurvivesFailedReopen pins that a failed rotation reopen (a
|
||||
// full filesystem, a permission change hit exactly when archied tries to
|
||||
// recreate the live file) does not take down the file sink permanently.
|
||||
// The write that triggered rotation must not be silently swallowed, and
|
||||
// -- unlike losing a single line -- every write after the failure must
|
||||
// not be lost either.
|
||||
func TestRotationSurvivesFailedReopen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "archied.log")
|
||||
|
||||
w, err := newRotatingFile(path, 1, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("newRotatingFile: %v", err)
|
||||
}
|
||||
|
||||
// Push size right up to the cap without writing megabytes of
|
||||
// fixture data -- white-box, same package -- so the marker write
|
||||
// below is guaranteed to be the one that crosses maxSize and
|
||||
// triggers rotate.
|
||||
marker := []byte("MARKER-LINE\n")
|
||||
w.size = w.maxSize - 1
|
||||
|
||||
// Simulate the reopen failing (disk full, permission change) while
|
||||
// the rename that frees the old name still succeeds -- the scenario
|
||||
// a chmod on the directory can't isolate, since that would also
|
||||
// block the rename.
|
||||
prevOpenFile := openFile
|
||||
openFile = func(name string, flag int, perm os.FileMode) (*os.File, error) {
|
||||
return nil, fmt.Errorf("simulated reopen failure")
|
||||
}
|
||||
t.Cleanup(func() { openFile = prevOpenFile })
|
||||
|
||||
if _, err := w.Write(marker); err != nil {
|
||||
t.Fatalf("Write after a failed reopen returned an error (line lost): %v", err)
|
||||
}
|
||||
|
||||
// A second write after the same failure must also land -- this is
|
||||
// what distinguishes "lost one line during rotation" from "the sink
|
||||
// is now permanently broken".
|
||||
marker2 := []byte("MARKER-LINE-2\n")
|
||||
if _, err := w.Write(marker2); err != nil {
|
||||
t.Fatalf("second write after a failed reopen returned an error: %v", err)
|
||||
}
|
||||
|
||||
openFile = prevOpenFile
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The live file was never successfully recreated (reopen kept
|
||||
// failing on every write), so each write's rotate attempt renamed
|
||||
// the same still-open descriptor further down the generation chain
|
||||
// -- the exact filename it lands under is incidental. What matters
|
||||
// is that neither marker is gone: scan the whole directory.
|
||||
var all strings.Builder
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all.Write(b)
|
||||
}
|
||||
if !strings.Contains(all.String(), "MARKER-LINE\n") {
|
||||
t.Error("MARKER-LINE missing after a failed reopen")
|
||||
}
|
||||
if !strings.Contains(all.String(), "MARKER-LINE-2\n") {
|
||||
t.Error("MARKER-LINE-2 missing -- the sink stayed broken after the transient failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRotationDefaults(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "archied.log")
|
||||
w, err := newRotatingFile(path, 0, 0)
|
||||
|
||||
Reference in new issue
Block a user