From ff3f80e4c1a7fcd9e50f4776e2ce0a4659731552 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 2 Oct 2026 10:33:24 +1000 Subject: [PATCH] fix(logging): file a task log record under its own event time A remotely-shipped task log entry was re-emitted through the sink with slog.Logger.LogAttrs, which stamps time.Now(): the entry's own event time was discarded and replaced with the daemon's receipt time. Because a container's records arrive in bursts, a multi-minute run collapsed onto a few milliseconds of receipt timestamps and its timing became unrecoverable -- which stage ran when, how long an iteration took, whether a stage stalled. Write now builds the record from entry.Time (falling back to receipt time only for an entry that carries none) and dispatches it through the same handler, preserving the sink's level gate. Receipt time stays available separately in the record's received_at field, so event time and arrival time cannot be confused. The agent-side producer already carried the record's own time; a test now pins that half too. --- internal/agentexec/systemlog_test.go | 12 ++++ internal/logging/registry.go | 44 ++++++++++++- internal/logging/registry_test.go | 98 ++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/internal/agentexec/systemlog_test.go b/internal/agentexec/systemlog_test.go index dd6ced53a..3be0e4db3 100644 --- a/internal/agentexec/systemlog_test.go +++ b/internal/agentexec/systemlog_test.go @@ -93,6 +93,18 @@ func TestSystemLogHandlerPublishesToTheTasksSystemSubject(t *testing.T) { if entry.Fields["component"] != "gate" { t.Errorf("Fields[component] = %v, want gate", entry.Fields["component"]) } + // The wire entry must carry the record's own event time. The daemon files + // a task's records under the time the event happened (see + // logging.TaskRegistry.Write), so a producer that dropped or re-stamped it + // here would collapse a run's timing before it ever left the container. + next.mu.Lock() + wantTime := next.records[0].Time + next.mu.Unlock() + if entry.Time.IsZero() { + t.Error("Entry.Time is the zero time, want the record's own event time") + } else if !entry.Time.Equal(wantTime) { + t.Errorf("Entry.Time = %s, want the record's event time %s", entry.Time, wantTime) + } } func TestSystemLogHandlerAlwaysTeesToNextRegardlessOfPublishOutcome(t *testing.T) { diff --git a/internal/logging/registry.go b/internal/logging/registry.go index b6c75e28c..62619e38f 100644 --- a/internal/logging/registry.go +++ b/internal/logging/registry.go @@ -5,8 +5,17 @@ import ( "log/slog" "os" "sync" + "time" ) +// receivedAtField is the record field carrying the receipt time of a +// remotely-shipped entry, alongside the entry's own event time in Time. +// Shipping and arriving are different moments -- a container's records cross +// NATS and can land in a burst, so a log that files them under arrival time +// cannot say when anything actually happened. Keeping both lets a reader +// measure delivery latency without losing the event's own timeline. +const receivedAtField = "received_at" + // TaskRegistry manages one open log destination per currently-running task // attempt, shared between whatever starts and ends a task's run (Open, // Close) and whatever receives that task's remotely-shipped log records @@ -83,7 +92,11 @@ func (r *TaskRegistry) Close(taskID int64) error { // Write appends one remotely-shipped entry to taskID's open sink, mirroring // it into the live dashboard feed the same way a locally-written log would -// be. Reports false for a task with no open sink -- not an error: a system +// be. The entry keeps the event time it was shipped with; the moment this +// process received it is recorded separately as the received_at field. +// Arrival time is never substituted for event time -- stamping on arrival is +// exactly what makes a run's timing unrecoverable. +// Reports false for a task with no open sink -- not an error: a system // log message is fire-and-forget best effort, and a late or duplicate // delivery after the task finished (or one this daemon instance never // dispatched) is expected, not exceptional. @@ -106,11 +119,36 @@ func (r *TaskRegistry) Write(ctx context.Context, taskID int64, entry Entry) boo var level slog.Level _ = level.UnmarshalText([]byte(entry.Level)) // unparseable falls back to Info, slog.Level's zero value - attrs := make([]slog.Attr, 0, len(entry.Fields)) + // File the record under the time the event happened, not the time this + // process received it. A run's records arrive in whatever bursts the + // transport produces, so stamping them on arrival collapses a + // multi-minute run onto a few milliseconds and makes its timing + // unrecoverable. An entry carrying no time of its own (the field is + // optional) is stamped now, so a line is never filed under the zero time. + received := time.Now() + happened := entry.Time + if happened.IsZero() { + happened = received + } + + attrs := make([]slog.Attr, 0, len(entry.Fields)+1) for k, v := range entry.Fields { attrs = append(attrs, slog.Any(k, v)) } - got.logger.LogAttrs(ctx, level, entry.Message, attrs...) + attrs = append(attrs, slog.Time(receivedAtField, received)) + + // slog.Logger.LogAttrs always stamps time.Now() and cannot be handed a + // record, so build the record here and dispatch it through the same + // handler the logger holds. The Enabled check is the logger's -- calling + // the handler directly skips it, which would write an entry below the + // sink's level to the file while the feed, gated separately, dropped it. + record := slog.NewRecord(happened, level, entry.Message, 0) + record.AddAttrs(attrs...) + handler := got.logger.Handler() + if !handler.Enabled(ctx, level) { + return true + } + _ = handler.Handle(ctx, record) return true } diff --git a/internal/logging/registry_test.go b/internal/logging/registry_test.go index cb49b24f9..6dcfaf052 100644 --- a/internal/logging/registry_test.go +++ b/internal/logging/registry_test.go @@ -1,9 +1,11 @@ package logging import ( + "fmt" "os" "sync" "testing" + "time" ) func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) { @@ -34,6 +36,102 @@ func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) { } } +// TestTaskRegistryWriteKeepsEachEntriesOwnEventTime guards the point of a +// task log: a record shipped from the container must be filed under the time +// the event happened, not the time the daemon happened to receive it. A run's +// iterations spread over minutes must be interleaved on disk the way they +// happened; collapsing them onto one receipt burst makes a task's timing +// unrecoverable (which stage ran when, how long an iteration took, whether a +// stage stalled). The daemon's receipt time is kept too, but separately -- in +// the record's own received_at field -- so "the event's time" and "when this +// process saw it" cannot be confused for one another. +func TestTaskRegistryWriteKeepsEachEntriesOwnEventTime(t *testing.T) { + baseDir := t.TempDir() + feed := NewFeed(10) + reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{}) + + if err := reg.Open(11, 1); err != nil { + t.Fatal(err) + } + // Three iterations, minutes apart -- the shape a real run has. + happened := []time.Time{ + time.Date(2026, 10, 1, 9, 30, 0, 123456789, time.UTC), + time.Date(2026, 10, 1, 9, 33, 41, 987654321, time.UTC), + time.Date(2026, 10, 1, 9, 37, 12, 500000000, time.UTC), + } + for i, at := range happened { + if ok := reg.Write(t.Context(), 11, Entry{Time: at, Level: "INFO", Message: fmt.Sprintf("iteration %d", i+1)}); !ok { + t.Fatalf("Write() = false for an open task, iteration %d", i) + } + } + if err := reg.Close(11); err != nil { + t.Fatal(err) + } + + result, err := Tail(TaskLogPath(baseDir, 11, 1), Query{}) + if err != nil { + t.Fatalf("Tail: %v", err) + } + if len(result.Entries) != len(happened) { + t.Fatalf("got %d entries, want %d", len(result.Entries), len(happened)) + } + for i, want := range happened { + entry := result.Entries[i] + if !entry.Time.Equal(want) { + t.Errorf("entry %d time = %s, want the iteration's own time %s", i, entry.Time, want) + } + if _, ok := entry.Fields[receivedAtField]; !ok { + t.Errorf("entry %d fields = %+v, want receipt time preserved separately as received_at", i, entry.Fields) + } + } + + // The live dashboard feed must carry the event's time too, not the + // mirror's own write time. + snapshot := feed.Snapshot() + if len(snapshot) != len(happened) { + t.Fatalf("feed has %d entries, want %d", len(snapshot), len(happened)) + } + for i, want := range happened { + if !snapshot[i].Time.Equal(want) { + t.Errorf("feed entry %d time = %s, want %s", i, snapshot[i].Time, want) + } + } +} + +// TestTaskRegistryWriteStampsAnEntryWithNoEventTime pins the fallback: a +// locally-produced entry that carries no time of its own (recordPark does set +// one, but the field is optional) is stamped now, so a log line is never +// filed under the zero time and never sinks to the bottom of a time-ordered +// view. +func TestTaskRegistryWriteStampsAnEntryWithNoEventTime(t *testing.T) { + baseDir := t.TempDir() + reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{}) + + if err := reg.Open(12, 1); err != nil { + t.Fatal(err) + } + before := time.Now() + if ok := reg.Write(t.Context(), 12, Entry{Level: "INFO", Message: "no time of its own"}); !ok { + t.Fatal("Write() = false for an open task") + } + after := time.Now() + if err := reg.Close(12); err != nil { + t.Fatal(err) + } + + result, err := Tail(TaskLogPath(baseDir, 12, 1), Query{}) + if err != nil { + t.Fatalf("Tail: %v", err) + } + if len(result.Entries) != 1 { + t.Fatalf("got %d entries, want 1", len(result.Entries)) + } + got := result.Entries[0].Time + if got.Before(before) || got.After(after) { + t.Errorf("entry time = %s, want a receipt-time stamp between %s and %s", got, before, after) + } +} + // TestTaskRegistryWriteMirrorsToTheDashboardFeed guards the other half of // the point of routing writes through a slog.Logger built on FeedHandler // instead of writing the file directly: a remotely-shipped entry must show