fix(logging): file a task log record under its own event time

A remotely-shipped task log entry was re-emitted through the sink with
slog.Logger.LogAttrs, which stamps time.Now(): the entry's own event time
was discarded and replaced with the daemon's receipt time. Because a
container's records arrive in bursts, a multi-minute run collapsed onto a
few milliseconds of receipt timestamps and its timing became
unrecoverable -- which stage ran when, how long an iteration took, whether
a stage stalled.

Write now builds the record from entry.Time (falling back to receipt time
only for an entry that carries none) and dispatches it through the same
handler, preserving the sink's level gate. Receipt time stays available
separately in the record's received_at field, so event time and arrival
time cannot be confused. The agent-side producer already carried the
record's own time; a test now pins that half too.
This commit is contained in:
sam committed 2026-10-02 10:33:24 +10:00
1 parent 148c81af36
commit cb923cc0d9
3 files changed
+151 -3

No files matched your search

+12
View File
@@ -93,6 +93,18 @@ func TestSystemLogHandlerPublishesToTheTasksSystemSubject(t *testing.T) {
if entry.Fields["component"] != "gate" {
t.Errorf("Fields[component] = %v, want gate", entry.Fields["component"])
}
// The wire entry must carry the record's own event time. The daemon files
// a task's records under the time the event happened (see
// logging.TaskRegistry.Write), so a producer that dropped or re-stamped it
// here would collapse a run's timing before it ever left the container.
next.mu.Lock()
wantTime := next.records[0].Time
next.mu.Unlock()
if entry.Time.IsZero() {
t.Error("Entry.Time is the zero time, want the record's own event time")
} else if !entry.Time.Equal(wantTime) {
t.Errorf("Entry.Time = %s, want the record's event time %s", entry.Time, wantTime)
}
}
func TestSystemLogHandlerAlwaysTeesToNextRegardlessOfPublishOutcome(t *testing.T) {
+41 -3
View File
@@ -5,8 +5,17 @@ import (
"log/slog"
"os"
"sync"
"time"
)
// receivedAtField is the record field carrying the receipt time of a
// remotely-shipped entry, alongside the entry's own event time in Time.
// Shipping and arriving are different moments -- a container's records cross
// NATS and can land in a burst, so a log that files them under arrival time
// cannot say when anything actually happened. Keeping both lets a reader
// measure delivery latency without losing the event's own timeline.
const receivedAtField = "received_at"
// TaskRegistry manages one open log destination per currently-running task
// attempt, shared between whatever starts and ends a task's run (Open,
// Close) and whatever receives that task's remotely-shipped log records
@@ -83,7 +92,11 @@ func (r *TaskRegistry) Close(taskID int64) error {
// Write appends one remotely-shipped entry to taskID's open sink, mirroring
// it into the live dashboard feed the same way a locally-written log would
// be. Reports false for a task with no open sink -- not an error: a system
// be. The entry keeps the event time it was shipped with; the moment this
// process received it is recorded separately as the received_at field.
// Arrival time is never substituted for event time -- stamping on arrival is
// exactly what makes a run's timing unrecoverable.
// Reports false for a task with no open sink -- not an error: a system
// log message is fire-and-forget best effort, and a late or duplicate
// delivery after the task finished (or one this daemon instance never
// dispatched) is expected, not exceptional.
@@ -106,11 +119,36 @@ func (r *TaskRegistry) Write(ctx context.Context, taskID int64, entry Entry) boo
var level slog.Level
_ = level.UnmarshalText([]byte(entry.Level)) // unparseable falls back to Info, slog.Level's zero value
attrs := make([]slog.Attr, 0, len(entry.Fields))
// File the record under the time the event happened, not the time this
// process received it. A run's records arrive in whatever bursts the
// transport produces, so stamping them on arrival collapses a
// multi-minute run onto a few milliseconds and makes its timing
// unrecoverable. An entry carrying no time of its own (the field is
// optional) is stamped now, so a line is never filed under the zero time.
received := time.Now()
happened := entry.Time
if happened.IsZero() {
happened = received
}
attrs := make([]slog.Attr, 0, len(entry.Fields)+1)
for k, v := range entry.Fields {
attrs = append(attrs, slog.Any(k, v))
}
got.logger.LogAttrs(ctx, level, entry.Message, attrs...)
attrs = append(attrs, slog.Time(receivedAtField, received))
// slog.Logger.LogAttrs always stamps time.Now() and cannot be handed a
// record, so build the record here and dispatch it through the same
// handler the logger holds. The Enabled check is the logger's -- calling
// the handler directly skips it, which would write an entry below the
// sink's level to the file while the feed, gated separately, dropped it.
record := slog.NewRecord(happened, level, entry.Message, 0)
record.AddAttrs(attrs...)
handler := got.logger.Handler()
if !handler.Enabled(ctx, level) {
return true
}
_ = handler.Handle(ctx, record)
return true
}
+98
View File
@@ -1,9 +1,11 @@
package logging
import (
"fmt"
"os"
"sync"
"testing"
"time"
)
func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) {
@@ -34,6 +36,102 @@ func TestTaskRegistryOpenWriteCloseRoundTripsThroughTail(t *testing.T) {
}
}
// TestTaskRegistryWriteKeepsEachEntriesOwnEventTime guards the point of a
// task log: a record shipped from the container must be filed under the time
// the event happened, not the time the daemon happened to receive it. A run's
// iterations spread over minutes must be interleaved on disk the way they
// happened; collapsing them onto one receipt burst makes a task's timing
// unrecoverable (which stage ran when, how long an iteration took, whether a
// stage stalled). The daemon's receipt time is kept too, but separately -- in
// the record's own received_at field -- so "the event's time" and "when this
// process saw it" cannot be confused for one another.
func TestTaskRegistryWriteKeepsEachEntriesOwnEventTime(t *testing.T) {
baseDir := t.TempDir()
feed := NewFeed(10)
reg := NewTaskRegistry(baseDir, feed, TaskSinkOptions{})
if err := reg.Open(11, 1); err != nil {
t.Fatal(err)
}
// Three iterations, minutes apart -- the shape a real run has.
happened := []time.Time{
time.Date(2026, 10, 1, 9, 30, 0, 123456789, time.UTC),
time.Date(2026, 10, 1, 9, 33, 41, 987654321, time.UTC),
time.Date(2026, 10, 1, 9, 37, 12, 500000000, time.UTC),
}
for i, at := range happened {
if ok := reg.Write(t.Context(), 11, Entry{Time: at, Level: "INFO", Message: fmt.Sprintf("iteration %d", i+1)}); !ok {
t.Fatalf("Write() = false for an open task, iteration %d", i)
}
}
if err := reg.Close(11); err != nil {
t.Fatal(err)
}
result, err := Tail(TaskLogPath(baseDir, 11, 1), Query{})
if err != nil {
t.Fatalf("Tail: %v", err)
}
if len(result.Entries) != len(happened) {
t.Fatalf("got %d entries, want %d", len(result.Entries), len(happened))
}
for i, want := range happened {
entry := result.Entries[i]
if !entry.Time.Equal(want) {
t.Errorf("entry %d time = %s, want the iteration's own time %s", i, entry.Time, want)
}
if _, ok := entry.Fields[receivedAtField]; !ok {
t.Errorf("entry %d fields = %+v, want receipt time preserved separately as received_at", i, entry.Fields)
}
}
// The live dashboard feed must carry the event's time too, not the
// mirror's own write time.
snapshot := feed.Snapshot()
if len(snapshot) != len(happened) {
t.Fatalf("feed has %d entries, want %d", len(snapshot), len(happened))
}
for i, want := range happened {
if !snapshot[i].Time.Equal(want) {
t.Errorf("feed entry %d time = %s, want %s", i, snapshot[i].Time, want)
}
}
}
// TestTaskRegistryWriteStampsAnEntryWithNoEventTime pins the fallback: a
// locally-produced entry that carries no time of its own (recordPark does set
// one, but the field is optional) is stamped now, so a log line is never
// filed under the zero time and never sinks to the bottom of a time-ordered
// view.
func TestTaskRegistryWriteStampsAnEntryWithNoEventTime(t *testing.T) {
baseDir := t.TempDir()
reg := NewTaskRegistry(baseDir, NewFeed(10), TaskSinkOptions{})
if err := reg.Open(12, 1); err != nil {
t.Fatal(err)
}
before := time.Now()
if ok := reg.Write(t.Context(), 12, Entry{Level: "INFO", Message: "no time of its own"}); !ok {
t.Fatal("Write() = false for an open task")
}
after := time.Now()
if err := reg.Close(12); err != nil {
t.Fatal(err)
}
result, err := Tail(TaskLogPath(baseDir, 12, 1), Query{})
if err != nil {
t.Fatalf("Tail: %v", err)
}
if len(result.Entries) != 1 {
t.Fatalf("got %d entries, want 1", len(result.Entries))
}
got := result.Entries[0].Time
if got.Before(before) || got.After(after) {
t.Errorf("entry time = %s, want a receipt-time stamp between %s and %s", got, before, after)
}
}
// TestTaskRegistryWriteMirrorsToTheDashboardFeed guards the other half of
// the point of routing writes through a slog.Logger built on FeedHandler
// instead of writing the file directly: a remotely-shipped entry must show