496 lines
15 KiB
Go
496 lines
15 KiB
Go
package logging
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Entry is one decoded log line.
|
|
type Entry struct {
|
|
ID int64 `json:"id,omitempty"`
|
|
Time time.Time `json:"time"`
|
|
Level string `json:"level"`
|
|
Message string `json:"msg"`
|
|
// Fields carries every remaining key so structured context survives the
|
|
// round trip -- component, task id, error and anything else a call site
|
|
// attached.
|
|
Fields map[string]any `json:"fields,omitempty"`
|
|
}
|
|
|
|
// Query filters a read. The zero value returns the most recent DefaultTailLines
|
|
// entries unfiltered.
|
|
type Query struct {
|
|
// Levels restricts results to these levels (case-insensitive). Empty
|
|
// means all levels.
|
|
Levels []string
|
|
// Component matches the "component" field exactly. Empty means any.
|
|
Component string
|
|
// Stage matches the entry's "stage" field, case-insensitively. Empty
|
|
// matches any. Entries without a stage never match.
|
|
Stage string
|
|
// Contains matches the message or any field value, case-insensitively.
|
|
Contains string
|
|
// Limit caps returned entries. Zero selects DefaultTailLines; values
|
|
// above MaxTailLines are clamped.
|
|
Limit int
|
|
// Since, when set, excludes entries whose Time is strictly before it.
|
|
// Until, when set, excludes entries whose Time is strictly after it.
|
|
// Both bounds are inclusive. The zero time disables its bound.
|
|
Since time.Time
|
|
Until time.Time
|
|
// BeforeID is the byte-offset cursor a previous page returned as Cursor: a
|
|
// read returns the most recent matching entries whose end offset is at or
|
|
// before it. Zero reads the tail of the scan window.
|
|
BeforeID int64
|
|
}
|
|
|
|
// Read bounds. A log file is unbounded input, so a request must not be able to
|
|
// pull an arbitrary amount of it into memory.
|
|
const (
|
|
DefaultTailLines = 200
|
|
MaxTailLines = 2000
|
|
// maxScanBytes caps how much of the tail is examined. Beyond this the
|
|
// result is reported as truncated rather than silently partial.
|
|
maxScanBytes = 8 << 20
|
|
)
|
|
|
|
// Result is a page of log entries, newest last.
|
|
type Result struct {
|
|
Entries []Entry `json:"entries"`
|
|
// Truncated reports that the scan hit maxScanBytes before satisfying the
|
|
// query, so older matching entries exist that were not examined.
|
|
Truncated bool `json:"truncated"`
|
|
// File is the path read, for the UI to show where this came from.
|
|
File string `json:"file"`
|
|
// Found reports whether the log file exists.
|
|
Found bool `json:"found"`
|
|
}
|
|
|
|
// TaskLogPage is one task attempt's log as a task-log reader returns it: the
|
|
// page Tail selected, whether the attempt has a file at all, and the distinct
|
|
// components present in the file's tail so a filter is built from what is
|
|
// actually there rather than a hardcoded list that drifts.
|
|
type TaskLogPage struct {
|
|
Result
|
|
Components []string `json:"components"`
|
|
// MoreAvailable reports matching entries did not fit in this page; Cursor
|
|
// is the byte offset to pass back as Query.BeforeID for the next older one.
|
|
MoreAvailable bool `json:"more_available"`
|
|
Cursor int64 `json:"cursor"`
|
|
}
|
|
|
|
// ErrTaskLogsUnavailable reports that this process cannot read task logs.
|
|
var ErrTaskLogsUnavailable = errors.New("logging: task log reader unavailable")
|
|
|
|
// taskLogChunkBytes bounds one write a TaskLogContent makes, so a reader that
|
|
// pushes into a bounded transport emits chunks that transport can carry.
|
|
// io.Copy would otherwise hand it reads of its own choosing.
|
|
const taskLogChunkBytes = 256 << 10
|
|
|
|
// TaskLog reads one task attempt's log. A nil registry returns
|
|
// ErrTaskLogsUnavailable.
|
|
func (r *TaskRegistry) TaskLog(_ context.Context, taskID int64, attempt int, q Query) (TaskLogPage, error) {
|
|
if r == nil {
|
|
return TaskLogPage{}, ErrTaskLogsUnavailable
|
|
}
|
|
res, err := PageBefore(TaskLogPath(r.baseDir, taskID, attempt), q, q.BeforeID)
|
|
if err != nil {
|
|
return TaskLogPage{}, err
|
|
}
|
|
page := TaskLogPage{
|
|
Entries: res.Entries, Truncated: res.Truncated, File: res.File, Found: res.Found,
|
|
Components: []string{},
|
|
MoreAvailable: res.MoreAvailable,
|
|
Cursor: res.Cursor,
|
|
}
|
|
// The filter list is a convenience: failing to build it must not cost the
|
|
// caller their entries. Components only reads the tail this page already
|
|
// read, so its error is the same "treat as empty" case the daemon-wide
|
|
// handler has always applied -- it is dropped deliberately, not missed.
|
|
if res.Found {
|
|
if components, err := Components(res.File); err == nil {
|
|
page.Components = components
|
|
}
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
// TaskLogContent streams one attempt's raw log to w. found is false when
|
|
// there is no file. A nil registry returns ErrTaskLogsUnavailable.
|
|
func (r *TaskRegistry) TaskLogContent(_ context.Context, taskID int64, attempt int, w io.Writer) (bool, error) {
|
|
if r == nil {
|
|
return false, ErrTaskLogsUnavailable
|
|
}
|
|
f, err := os.Open(TaskLogPath(r.baseDir, taskID, attempt))
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return false, nil
|
|
}
|
|
return false, fmt.Errorf("logging: open task log: %w", err)
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
if _, err := io.CopyBuffer(w, f, make([]byte, taskLogChunkBytes)); err != nil {
|
|
return false, fmt.Errorf("logging: read task log: %w", err)
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// PageResult is one forward page. Pass Cursor back to continue.
|
|
// MoreAvailable reports matches beyond Limit.
|
|
type PageResult struct {
|
|
Entries []Entry `json:"entries"`
|
|
Truncated bool `json:"truncated"`
|
|
MoreAvailable bool `json:"more_available"`
|
|
Cursor int64 `json:"cursor"`
|
|
File string `json:"file"`
|
|
// Found reports whether the log file exists; false with no entries is an
|
|
// attempt with no log, not an error.
|
|
Found bool `json:"found"`
|
|
}
|
|
|
|
// logWindow is the region of a log file one read examines: the
|
|
// tail-most maxScanBytes, held in memory so offset arithmetic is
|
|
// against a contiguous buffer rather than a live file position.
|
|
type logWindow struct {
|
|
// data is the window's bytes; data[0] is at file offset start.
|
|
data []byte
|
|
// start is the file offset data begins at (0 when the whole file fits).
|
|
start int64
|
|
// fileSize is the file's size when the window was taken.
|
|
fileSize int64
|
|
// truncated reports that data is a strict suffix of the file, so
|
|
// older bytes exist that this window does not contain.
|
|
truncated bool
|
|
}
|
|
|
|
// readWindow opens path and reads its scan window into memory.
|
|
// found is false (with a nil error) for an absent file or empty path:
|
|
// file logging is optional, so neither is an error condition.
|
|
func readWindow(path string) (w logWindow, found bool, err error) {
|
|
if strings.TrimSpace(path) == "" {
|
|
return logWindow{}, false, nil
|
|
}
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return logWindow{}, false, nil
|
|
}
|
|
return logWindow{}, false, fmt.Errorf("logging: open %s: %w", path, err)
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
info, err := f.Stat()
|
|
if err != nil {
|
|
return logWindow{}, false, fmt.Errorf("logging: stat %s: %w", path, err)
|
|
}
|
|
|
|
w = logWindow{fileSize: info.Size()}
|
|
size := info.Size()
|
|
if size > maxScanBytes {
|
|
w.start = size - maxScanBytes
|
|
size = maxScanBytes
|
|
w.truncated = true
|
|
}
|
|
w.data = make([]byte, size)
|
|
|
|
// ReadAt fills the buffer or errors; skip it for an empty window.
|
|
if len(w.data) > 0 {
|
|
if _, err := f.ReadAt(w.data, w.start); err != nil {
|
|
return logWindow{}, false, fmt.Errorf("logging: read %s: %w", path, err)
|
|
}
|
|
}
|
|
return w, true, nil
|
|
}
|
|
|
|
// readLines calls step for each matching entry in the last maxScanBytes of
|
|
// path, with the offset after its line. It returns whether the window
|
|
// excluded older data and the next cursor. Older data is never read.
|
|
func readLines(path string, q Query, cursor int64, step func(e Entry, startOff, endOff int64) bool) (truncated bool, cursorOut int64, ok bool, err error) {
|
|
w, found, err := readWindow(path)
|
|
if err != nil || !found {
|
|
return false, 0, false, err
|
|
}
|
|
truncated = w.truncated
|
|
windowStart, data := w.start, w.data
|
|
|
|
// Clamp the caller's cursor to the scan window. A cursor below
|
|
// windowStart cannot read older history than Tail would have shown;
|
|
// we clamp upward so the returned cursor stays inside the window.
|
|
if cursor < windowStart {
|
|
cursor = windowStart
|
|
}
|
|
// A cursor at or past EOF means the caller has read everything we
|
|
// can serve. Return ok=false so the caller stops looping; the
|
|
// preserved cursor lets it detect overshoot (file shrank under
|
|
// rotation) versus exact-fit.
|
|
if cursor >= w.fileSize {
|
|
return truncated, cursor, false, nil
|
|
}
|
|
|
|
// Convert the file cursor to an offset in data.
|
|
cursorInWindow := min(cursor-windowStart, int64(len(data)))
|
|
|
|
// Skip a partial first line at the window start.
|
|
if cursor == windowStart && truncated {
|
|
idx := bytes.IndexByte(data, '\n')
|
|
if idx < 0 {
|
|
return truncated, w.fileSize, false, nil
|
|
}
|
|
cursorInWindow = int64(idx + 1)
|
|
}
|
|
|
|
return truncated, walkWindow(data, windowStart, cursorInWindow, q, step), true, nil
|
|
}
|
|
|
|
// walkWindow calls step for each matching entry from offset from, skipping
|
|
// undecodable lines, and returns the file offset to resume at. step returns
|
|
// false to stop.
|
|
func walkWindow(data []byte, windowStart, from int64, q Query, step func(e Entry, startOff, endOff int64) bool) int64 {
|
|
pos := from
|
|
for pos < int64(len(data)) {
|
|
// lineEnd indexes the line's terminating '\n'; when the final
|
|
// line is unterminated (daemon killed mid-write) it indexes the
|
|
// end of the buffer and endOff stops there.
|
|
nl := bytes.IndexByte(data[pos:], '\n')
|
|
lineEnd := int64(len(data))
|
|
endOff := windowStart + int64(len(data))
|
|
if nl >= 0 {
|
|
lineEnd = pos + int64(nl)
|
|
endOff = windowStart + lineEnd + 1 // past the '\n'
|
|
}
|
|
raw := data[pos:lineEnd] // excludes the '\n'
|
|
if len(raw) > 0 && raw[len(raw)-1] == '\r' {
|
|
raw = raw[:len(raw)-1] // CRLF
|
|
}
|
|
if entry, decoded := decode(raw); decoded && q.matches(entry) {
|
|
entry.ID = endOff
|
|
if !step(entry, windowStart+pos, endOff) {
|
|
return endOff
|
|
}
|
|
}
|
|
if nl < 0 {
|
|
return windowStart + int64(len(data))
|
|
}
|
|
pos = lineEnd + 1
|
|
}
|
|
return windowStart + pos
|
|
}
|
|
|
|
// Tail returns the most recent matching entries in path, oldest first. A
|
|
// missing file is not an error. Past Limit, older matches are dropped and
|
|
// Truncated is set.
|
|
func Tail(path string, q Query) (Result, error) {
|
|
res := Result{Entries: []Entry{}, File: path}
|
|
limit := q.Limit
|
|
if limit <= 0 {
|
|
limit = DefaultTailLines
|
|
}
|
|
if limit > MaxTailLines {
|
|
limit = MaxTailLines
|
|
}
|
|
|
|
// Stat for Found without reading the window twice.
|
|
if strings.TrimSpace(path) != "" {
|
|
if _, err := os.Stat(path); err == nil {
|
|
res.Found = true
|
|
} else if !os.IsNotExist(err) {
|
|
return Result{}, fmt.Errorf("logging: stat %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
matches := make([]Entry, 0, limit)
|
|
truncated, _, ok, err := readLines(path, q, 0, func(e Entry, _, _ int64) bool {
|
|
if len(matches) == limit {
|
|
matches = append(matches[:0], matches[1:]...)
|
|
res.Truncated = true
|
|
}
|
|
matches = append(matches, e)
|
|
return true
|
|
})
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
if !ok {
|
|
// Truncated stays set even with no entries.
|
|
res.Truncated = truncated
|
|
return res, nil
|
|
}
|
|
res.Truncated = res.Truncated || truncated
|
|
res.Entries = matches
|
|
return res, nil
|
|
}
|
|
|
|
// PageBefore returns up to Limit matching entries whose end offset is at or
|
|
// before before (the whole scan window when before is zero), oldest first. It
|
|
// is the backward counterpart of Page: Cursor is the before value for the next
|
|
// older page, and MoreAvailable reports older matches beyond this page.
|
|
func PageBefore(path string, q Query, before int64) (PageResult, error) {
|
|
res := PageResult{Entries: []Entry{}, File: path}
|
|
limit := q.Limit
|
|
if limit <= 0 {
|
|
limit = DefaultTailLines
|
|
}
|
|
if limit > MaxTailLines {
|
|
limit = MaxTailLines
|
|
}
|
|
|
|
// Stat for Found without reading the window twice. readLines cannot report
|
|
// the difference between a missing file and a cursor at EOF.
|
|
if strings.TrimSpace(path) != "" {
|
|
if _, err := os.Stat(path); err == nil {
|
|
res.Found = true
|
|
} else if !os.IsNotExist(err) {
|
|
return PageResult{}, fmt.Errorf("logging: stat %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
type lineAt struct {
|
|
entry Entry
|
|
start, end int64
|
|
}
|
|
matches := make([]lineAt, 0, limit)
|
|
truncated, _, ok, err := readLines(path, q, 0, func(e Entry, start, end int64) bool {
|
|
// File order is oldest to newest: once past the bound, nothing older
|
|
// follows.
|
|
if before > 0 && end > before {
|
|
return false
|
|
}
|
|
if len(matches) == limit {
|
|
// Drop the oldest so the page holds the limit entries closest to
|
|
// before, and remember older entries exist beyond it.
|
|
matches = append(matches[:0], matches[1:]...)
|
|
res.MoreAvailable = true
|
|
}
|
|
matches = append(matches, lineAt{entry: e, start: start, end: end})
|
|
return true
|
|
})
|
|
if err != nil {
|
|
return PageResult{}, err
|
|
}
|
|
if !ok {
|
|
res.Truncated = truncated
|
|
return res, nil
|
|
}
|
|
res.Truncated = truncated
|
|
res.Entries = make([]Entry, len(matches))
|
|
for i, m := range matches {
|
|
res.Entries[i] = m.entry
|
|
}
|
|
// The oldest returned entry's start is the before value that returns the
|
|
// entries strictly older than this page. No matches means no older page.
|
|
if len(matches) > 0 {
|
|
res.Cursor = matches[0].start
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
// decode parses one slog JSON line, lifting the known keys out and keeping the
|
|
// rest as structured fields. A line that is not JSON is skipped: a log file may
|
|
// contain output from a subprocess that does not share our format.
|
|
func decode(line []byte) (Entry, bool) {
|
|
var raw map[string]any
|
|
if err := json.Unmarshal(line, &raw); err != nil {
|
|
return Entry{}, false
|
|
}
|
|
|
|
entry := Entry{Fields: map[string]any{}}
|
|
for k, v := range raw {
|
|
switch k {
|
|
case "time":
|
|
if s, ok := v.(string); ok {
|
|
entry.Time, _ = time.Parse(time.RFC3339Nano, s)
|
|
}
|
|
case "level":
|
|
entry.Level, _ = v.(string)
|
|
case "msg":
|
|
entry.Message, _ = v.(string)
|
|
default:
|
|
entry.Fields[k] = v
|
|
}
|
|
}
|
|
if len(entry.Fields) == 0 {
|
|
entry.Fields = nil
|
|
}
|
|
return entry, true
|
|
}
|
|
|
|
// withinTimeBounds reports whether t is within q's inclusive Since/Until
|
|
// bounds. An entry with no parsed time fails any bound.
|
|
func (q Query) withinTimeBounds(t time.Time) bool {
|
|
if q.Since.IsZero() && q.Until.IsZero() {
|
|
return true
|
|
}
|
|
if t.IsZero() {
|
|
return false
|
|
}
|
|
if !q.Since.IsZero() && t.Before(q.Since) {
|
|
return false
|
|
}
|
|
return q.Until.IsZero() || !t.After(q.Until)
|
|
}
|
|
|
|
func (q Query) matches(e Entry) bool {
|
|
if len(q.Levels) > 0 && !slices.ContainsFunc(q.Levels, func(l string) bool {
|
|
return strings.EqualFold(l, e.Level)
|
|
}) {
|
|
return false
|
|
}
|
|
if q.Component != "" {
|
|
got, _ := e.Fields["component"].(string)
|
|
if !strings.EqualFold(got, q.Component) {
|
|
return false
|
|
}
|
|
}
|
|
if q.Stage != "" {
|
|
got, _ := e.Fields["stage"].(string)
|
|
if !strings.EqualFold(got, q.Stage) {
|
|
return false
|
|
}
|
|
}
|
|
if !q.withinTimeBounds(e.Time) {
|
|
return false
|
|
}
|
|
if q.Contains != "" {
|
|
needle := strings.ToLower(q.Contains)
|
|
if strings.Contains(strings.ToLower(e.Message), needle) {
|
|
return true
|
|
}
|
|
for _, v := range e.Fields {
|
|
if strings.Contains(strings.ToLower(fmt.Sprint(v)), needle) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Components returns the distinct "component" values present in the tail, so
|
|
// the UI can offer a filter built from what is actually there rather than a
|
|
// hardcoded list that drifts.
|
|
func Components(path string) ([]string, error) {
|
|
res, err := Tail(path, Query{Limit: MaxTailLines})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
seen := map[string]struct{}{}
|
|
out := []string{}
|
|
for _, e := range res.Entries {
|
|
if c, ok := e.Fields["component"].(string); ok && c != "" {
|
|
if _, dup := seen[c]; !dup {
|
|
seen[c] = struct{}{}
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
}
|
|
slices.Sort(out)
|
|
return out, nil
|
|
}
|