From bb4e2f91356fc702968e701a02212a1e21787a00 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Tue, 15 Sep 2026 04:55:48 +1000 Subject: [PATCH] fix(logging): keep the live descriptor open across a failed rotation rotate() closed the current file before attempting the rename+reopen, so when the reopen failed (a full disk, a permission change) every write afterward went to a closed *os.File and was silently dropped -- the sink stayed permanently broken rather than recovering once the transient condition cleared. Keep the old descriptor open across the rename/reopen and only close it once the new one is confirmed, so a write during (or after) a failed rotation always lands somewhere. --- internal/logging/logging.go | 30 ++++++++++--- internal/logging/logging_test.go | 74 ++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/internal/logging/logging.go b/internal/logging/logging.go index 6c4188b18..4e38d31c8 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -125,8 +125,13 @@ func newRotatingFile(path string, maxSizeMB, keep int) (*rotatingFile, error) { return r, nil } +// openFile is os.OpenFile by default; tests override it to inject a +// deterministic reopen failure without needing filesystem-permission +// tricks that a failed rename would also trip. +var openFile = os.OpenFile + func (r *rotatingFile) open() error { - f, err := os.OpenFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) + f, err := openFile(r.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o640) if err != nil { return err } @@ -153,18 +158,29 @@ func (r *rotatingFile) Write(p []byte) (int, error) { } // rotate renames the live file to .1, shifting existing generations down and -// discarding anything past keep. +// discarding anything past keep, then opens a fresh file at path. +// +// The live descriptor is kept open across the renames -- POSIX rename only +// changes a directory entry, not what an open file descriptor points at -- +// so if the rename or the reopen below fails, r.f still refers to a +// perfectly writable file (the original, or the just-rotated one) and the +// caller's write is never lost. Closing the old descriptor only happens +// once the new one is confirmed open. func (r *rotatingFile) rotate() error { - if err := r.f.Close(); err != nil { - return err - } // Oldest first, so each rename lands on a free slot. _ = os.Remove(fmt.Sprintf("%s.%d", r.path, r.keep)) for i := r.keep - 1; i >= 1; i-- { _ = os.Rename(fmt.Sprintf("%s.%d", r.path, i), fmt.Sprintf("%s.%d", r.path, i+1)) } - _ = os.Rename(r.path, r.path+".1") - return r.open() + if err := os.Rename(r.path, r.path+".1"); err != nil { + return err + } + old := r.f + if err := r.open(); err != nil { + r.f = old + return err + } + return old.Close() } func (r *rotatingFile) Close() error { diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go index cebffb567..9edb6f90a 100644 --- a/internal/logging/logging_test.go +++ b/internal/logging/logging_test.go @@ -156,6 +156,80 @@ func TestRotation(t *testing.T) { } } +// TestRotationSurvivesFailedReopen pins that a failed rotation reopen (a +// full filesystem, a permission change hit exactly when archied tries to +// recreate the live file) does not take down the file sink permanently. +// The write that triggered rotation must not be silently swallowed, and +// -- unlike losing a single line -- every write after the failure must +// not be lost either. +func TestRotationSurvivesFailedReopen(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "archied.log") + + w, err := newRotatingFile(path, 1, 2) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + + // Push size right up to the cap without writing megabytes of + // fixture data -- white-box, same package -- so the marker write + // below is guaranteed to be the one that crosses maxSize and + // triggers rotate. + marker := []byte("MARKER-LINE\n") + w.size = w.maxSize - 1 + + // Simulate the reopen failing (disk full, permission change) while + // the rename that frees the old name still succeeds -- the scenario + // a chmod on the directory can't isolate, since that would also + // block the rename. + prevOpenFile := openFile + openFile = func(name string, flag int, perm os.FileMode) (*os.File, error) { + return nil, fmt.Errorf("simulated reopen failure") + } + t.Cleanup(func() { openFile = prevOpenFile }) + + if _, err := w.Write(marker); err != nil { + t.Fatalf("Write after a failed reopen returned an error (line lost): %v", err) + } + + // A second write after the same failure must also land -- this is + // what distinguishes "lost one line during rotation" from "the sink + // is now permanently broken". + marker2 := []byte("MARKER-LINE-2\n") + if _, err := w.Write(marker2); err != nil { + t.Fatalf("second write after a failed reopen returned an error: %v", err) + } + + openFile = prevOpenFile + if err := w.Close(); err != nil { + t.Fatal(err) + } + + // The live file was never successfully recreated (reopen kept + // failing on every write), so each write's rotate attempt renamed + // the same still-open descriptor further down the generation chain + // -- the exact filename it lands under is incidental. What matters + // is that neither marker is gone: scan the whole directory. + var all strings.Builder + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + t.Fatal(err) + } + all.Write(b) + } + if !strings.Contains(all.String(), "MARKER-LINE\n") { + t.Error("MARKER-LINE missing after a failed reopen") + } + if !strings.Contains(all.String(), "MARKER-LINE-2\n") { + t.Error("MARKER-LINE-2 missing -- the sink stayed broken after the transient failure") + } +} + func TestRotationDefaults(t *testing.T) { path := filepath.Join(t.TempDir(), "archied.log") w, err := newRotatingFile(path, 0, 0)