diff --git a/internal/agentexec/systemlog_test.go b/internal/agentexec/systemlog_test.go index dd6ced53a..3be0e4db3 100644 --- a/internal/agentexec/systemlog_test.go +++ b/internal/agentexec/systemlog_test.go @@ -93,6 +93,18 @@ func TestSystemLogHandlerPublishesToTheTasksSystemSubject(t *testing.T) { if entry.Fields["component"] != "gate" { t.Errorf("Fields[component] = %v, want gate", entry.Fields["component"]) } + // The wire entry must carry the record's own event time. The daemon files + // a task's records under the time the event happened (see + // logging.TaskRegistry.Write), so a producer that dropped or re-stamped it + // here would collapse a run's timing before it ever left the container. + next.mu.Lock() + wantTime := next.records[0].Time + next.mu.Unlock() + if entry.Time.IsZero() { + t.Error("Entry.Time is the zero time, want the record's own event time") + } else if !entry.Time.Equal(wantTime) { + t.Errorf("Entry.Time = %s, want the record's event time %s", entry.Time, wantTime) + } } func TestSystemLogHandlerAlwaysTeesToNextRegardlessOfPublishOutcome(t *testing.T) { diff --git a/internal/logging/registry.go b/internal/logging/registry.go index b6c75e28c..62619e38f 100644 --- a/internal/logging/registry.go +++ b/internal/logging/registry.go @@ -5,8 +5,17 @@ import ( "log/slog" "os" "sync" + "time" ) +// receivedAtField is the record field carrying the receipt time of a +// remotely-shipped entry, alongside the entry's own event time in Time. +// Shipping and arriving are different moments -- a container's records cross +// NATS and can land in a burst, so a log that files them under arrival time +// cannot say when anything actually happened. Keeping both lets a reader +// measure delivery latency without losing the event's own timeline. +const receivedAtField = "received_at" + // TaskRegistry manages one open log destination per currently-running task // attempt, shared between whatever starts and ends a task's run (Open, // Close) and whatever receives that task's remotely-shipped log records @@ -83,7 +92,11 @@ func (r *TaskRegistry) Close(taskID int64) error { // Write appends one remotely-shipped entry to taskID's open sink, mirroring // it into the live dashboard feed the same way a locally-written log would -// be. Reports false for a task with no open sink -- not an error: a system +// be. The entry keeps the event time it was shipped with; the moment this +// process received it is recorded separately as the received_at field. +// Arrival time is never substituted for event time -- stamping on arrival is +// exactly what makes a run's timing unrecoverable. +// Reports false for a task with no open sink -- not an error: a system // log message is fire-and-forget best effort, and a late or duplicate // delivery after the task finished (or one this daemon instance never // dispatched) is expected, not exceptional. @@ -106,11 +119,36 @@ func (r *TaskRegistry) Write(ctx context.Context, taskID int64, entry Entry) boo var level slog.Level _ = level.UnmarshalText([]byte(entry.Level)) // unparseable falls back to Info, slog.Level's zero value - attrs := make([]slog.Attr, 0, len(entry.Fields)) + // File the record under the time the event happened, not the time this + // process received it. A run's records arrive in whatever bursts the + // transport produces, so stamping them on arrival collapses a + // multi-minute run onto a few milliseconds and makes its timing + // unrecoverable. An entry carrying no time of its own (the field is + // optional) is stamped now, so a line is never filed under the zero time. + received := time.Now() + happened := entry.Time + if happened.IsZero() { + happened = received + } + + attrs := make([]slog.Attr, 0, len(entry.Fields)+1) for k, v := range entry.Fields { attrs = append(attrs, slog.Any(k, v)) } - got.logger.LogAttrs(ctx, level, entry.Message, attrs...) + attrs = append(attrs, slog.Time(receivedAtField, received)) + + // slog.Logger.LogAttrs always stamps time.Now() and cannot be handed a + // record, so build the record here and dispatch it through the same + // handler the logger holds. The Enabled check is the logger's -- calling + // the handler directly skips it, which would write an entry below the + // sink's level to the file while the feed, gated separately, dropped it. + record := slog.NewRecord(happened, level, entry.Message, 0) + record.AddAttrs(attrs...) + handler := got.logger.Handler() + if !handler.Enabled(ctx, level) { + return true + } + _ = handler.Handle(ctx, record) return true } diff --git a/internal/logging/registry_test.go b/internal/logging/registry_test.go index cb49b24f9..6dcfaf052 100644 --- a/internal/logging/registry_test.go +++ b/internal/logging/registry_test.go @@ -1,9 +1,11 @@ package logging import ( + "fmt" "os" "sync" "testing" + "time" ) func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) { @@ -34,6 +36,102 @@ func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) { } } +// TestTaskRegistryWriteKeepsEachEntriesOwnEventTime guards the point of a +// task log: a record shipped from the container must be filed under the time +// the event happened, not the time the daemon happened to receive it. A run's +// iterations spread over minutes must be interleaved on disk the way they +// happened; collapsing them onto one receipt burst makes a task's timing +// unrecoverable (which stage ran when, how long an iteration took, whether a +// stage stalled). The daemon's receipt time is kept too, but separately -- in +// the record's own received_at field -- so "the event's time" and "when this +// process saw it" cannot be confused for one another. +func TestTaskRegistryWriteKeepsEachEntriesOwnEventTime(t *testing.T) { + baseDir := t.TempDir() + feed := NewFeed(10) + reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{}) + + if err := reg.Open(11, 1); err != nil { + t.Fatal(err) + } + // Three iterations, minutes apart -- the shape a real run has. + happened := []time.Time{ + time.Date(2026, 10, 1, 9, 30, 0, 123456789, time.UTC), + time.Date(2026, 10, 1, 9, 33, 41, 987654321, time.UTC), + time.Date(2026, 10, 1, 9, 37, 12, 500000000, time.UTC), + } + for i, at := range happened { + if ok := reg.Write(t.Context(), 11, Entry{Time: at, Level: "INFO", Message: fmt.Sprintf("iteration %d", i+1)}); !ok { + t.Fatalf("Write() = false for an open task, iteration %d", i) + } + } + if err := reg.Close(11); err != nil { + t.Fatal(err) + } + + result, err := Tail(TaskLogPath(baseDir, 11, 1), Query{}) + if err != nil { + t.Fatalf("Tail: %v", err) + } + if len(result.Entries) != len(happened) { + t.Fatalf("got %d entries, want %d", len(result.Entries), len(happened)) + } + for i, want := range happened { + entry := result.Entries[i] + if !entry.Time.Equal(want) { + t.Errorf("entry %d time = %s, want the iteration's own time %s", i, entry.Time, want) + } + if _, ok := entry.Fields[receivedAtField]; !ok { + t.Errorf("entry %d fields = %+v, want receipt time preserved separately as received_at", i, entry.Fields) + } + } + + // The live dashboard feed must carry the event's time too, not the + // mirror's own write time. + snapshot := feed.Snapshot() + if len(snapshot) != len(happened) { + t.Fatalf("feed has %d entries, want %d", len(snapshot), len(happened)) + } + for i, want := range happened { + if !snapshot[i].Time.Equal(want) { + t.Errorf("feed entry %d time = %s, want %s", i, snapshot[i].Time, want) + } + } +} + +// TestTaskRegistryWriteStampsAnEntryWithNoEventTime pins the fallback: a +// locally-produced entry that carries no time of its own (recordPark does set +// one, but the field is optional) is stamped now, so a log line is never +// filed under the zero time and never sinks to the bottom of a time-ordered +// view. +func TestTaskRegistryWriteStampsAnEntryWithNoEventTime(t *testing.T) { + baseDir := t.TempDir() + reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{}) + + if err := reg.Open(12, 1); err != nil { + t.Fatal(err) + } + before := time.Now() + if ok := reg.Write(t.Context(), 12, Entry{Level: "INFO", Message: "no time of its own"}); !ok { + t.Fatal("Write() = false for an open task") + } + after := time.Now() + if err := reg.Close(12); err != nil { + t.Fatal(err) + } + + result, err := Tail(TaskLogPath(baseDir, 12, 1), Query{}) + if err != nil { + t.Fatalf("Tail: %v", err) + } + if len(result.Entries) != 1 { + t.Fatalf("got %d entries, want 1", len(result.Entries)) + } + got := result.Entries[0].Time + if got.Before(before) || got.After(after) { + t.Errorf("entry time = %s, want a receipt-time stamp between %s and %s", got, before, after) + } +} + // TestTaskRegistryWriteMirrorsToTheDashboardFeed guards the other half of // the point of routing writes through a slog.Logger built on FeedHandler // instead of writing the file directly: a remotely-shipped entry must show