145 lines
4.5 KiB
Go
145 lines
4.5 KiB
Go
// Package spanrelay carries spans from task containers to the collector. A
|
|
// container has no route to the collector, so its tracer provider exports as
|
|
// OTLP messages over JetStream and the daemon forwards them upstream.
|
|
package spanrelay
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
|
coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
|
|
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
"github.com/samcharles93/archie-core/internal/eventbus"
|
|
)
|
|
|
|
const (
|
|
// StreamName holds spans in flight. It is separate from ARCHIE_TASKS,
|
|
// which is a work queue.
|
|
StreamName = "ARCHIE_SPANS"
|
|
// ConsumerName is the daemon's one durable consumer.
|
|
ConsumerName = "archie-spans"
|
|
// SubjectWildcard matches every task's span subject.
|
|
SubjectWildcard = "archie.spans.>"
|
|
// MaxAge bounds how long an undelivered batch is kept.
|
|
MaxAge = time.Hour
|
|
|
|
publishTimeout = 5 * time.Second
|
|
uploadTimeout = 10 * time.Second
|
|
retryDelay = 2 * time.Second
|
|
)
|
|
|
|
// Subject is where a task's container publishes its span batches.
|
|
func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) }
|
|
|
|
// Exporter is an otlptrace.Client that publishes each batch to the span
|
|
// stream and returns once the stream has acknowledged it.
|
|
type Exporter struct {
|
|
conn *nats.Conn
|
|
subject string
|
|
}
|
|
|
|
var _ otlptrace.Client = (*Exporter)(nil)
|
|
|
|
// NewExporter publishes taskID's spans over conn.
|
|
func NewExporter(conn *nats.Conn, taskID int64) *Exporter {
|
|
return &Exporter{conn: conn, subject: Subject(taskID)}
|
|
}
|
|
|
|
// Start is a no-op: the connection is already open.
|
|
func (*Exporter) Start(context.Context) error { return nil }
|
|
|
|
// Stop is a no-op: the connection belongs to the transport.
|
|
func (*Exporter) Stop(context.Context) error { return nil }
|
|
|
|
// UploadTraces publishes one batch and waits for the JetStream ack.
|
|
func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
|
|
data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans})
|
|
if err != nil {
|
|
return fmt.Errorf("encode spans: %w", err)
|
|
}
|
|
if limit := e.conn.MaxPayload(); int64(len(data)) > limit {
|
|
return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit)
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, publishTimeout)
|
|
defer cancel()
|
|
msg := nats.NewMsg(e.subject)
|
|
msg.Data = data
|
|
reply, err := e.conn.RequestMsgWithContext(ctx, msg)
|
|
if err != nil {
|
|
return fmt.Errorf("publish spans: %w", err)
|
|
}
|
|
var ack struct {
|
|
Error *struct {
|
|
Description string `json:"description"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
|
return fmt.Errorf("decode span ack: %w", err)
|
|
}
|
|
if ack.Error != nil {
|
|
return fmt.Errorf("publish spans: %s", ack.Error.Description)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Relay forwards span batches from consumer to upstream until ctx ends. A
|
|
// batch that cannot be forwarded is returned to the stream and retried until
|
|
// the consumer's delivery limit drops it.
|
|
func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) {
|
|
if err := upstream.Start(ctx); err != nil {
|
|
log.ErrorContext(ctx, "span relay upstream start failed", "err", err)
|
|
return
|
|
}
|
|
defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }()
|
|
for ctx.Err() == nil {
|
|
msg, err := consumer.Fetch(ctx)
|
|
if errors.Is(err, eventbus.ErrNoMessage) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
if ctx.Err() == nil {
|
|
log.WarnContext(ctx, "span relay fetch failed", "err", err)
|
|
sleep(ctx, retryDelay)
|
|
}
|
|
continue
|
|
}
|
|
forward(ctx, msg, upstream, log)
|
|
}
|
|
}
|
|
|
|
func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) {
|
|
var req coltracepb.ExportTraceServiceRequest
|
|
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
|
|
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
|
|
_ = msg.Ack()
|
|
return
|
|
}
|
|
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
|
|
defer cancel()
|
|
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
|
|
log.WarnContext(ctx, "span relay upload failed", "err", err)
|
|
_ = msg.Nak()
|
|
sleep(ctx, retryDelay)
|
|
return
|
|
}
|
|
_ = msg.Ack()
|
|
}
|
|
|
|
func sleep(ctx context.Context, d time.Duration) {
|
|
t := time.NewTimer(d)
|
|
defer t.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-t.C:
|
|
}
|
|
}
|