Files
archie/internal/infrastructure/spanrelay/spanrelay.go
T

145 lines
4.5 KiB
Go

// Package spanrelay carries spans from task containers to the collector. A
// container has no route to the collector, so its tracer provider exports as
// OTLP messages over JetStream and the daemon forwards them upstream.
package spanrelay
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strconv"
"time"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
"google.golang.org/protobuf/proto"
"github.com/samcharles93/archie-core/internal/eventbus"
)
const (
// StreamName holds spans in flight. It is separate from ARCHIE_TASKS,
// which is a work queue.
StreamName = "ARCHIE_SPANS"
// ConsumerName is the daemon's one durable consumer.
ConsumerName = "archie-spans"
// SubjectWildcard matches every task's span subject.
SubjectWildcard = "archie.spans.>"
// MaxAge bounds how long an undelivered batch is kept.
MaxAge = time.Hour
publishTimeout = 5 * time.Second
uploadTimeout = 10 * time.Second
retryDelay = 2 * time.Second
)
// Subject is where a task's container publishes its span batches.
func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) }
// Exporter is an otlptrace.Client that publishes each batch to the span
// stream and returns once the stream has acknowledged it.
type Exporter struct {
conn *nats.Conn
subject string
}
var _ otlptrace.Client = (*Exporter)(nil)
// NewExporter publishes taskID's spans over conn.
func NewExporter(conn *nats.Conn, taskID int64) *Exporter {
return &Exporter{conn: conn, subject: Subject(taskID)}
}
// Start is a no-op: the connection is already open.
func (*Exporter) Start(context.Context) error { return nil }
// Stop is a no-op: the connection belongs to the transport.
func (*Exporter) Stop(context.Context) error { return nil }
// UploadTraces publishes one batch and waits for the JetStream ack.
func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans})
if err != nil {
return fmt.Errorf("encode spans: %w", err)
}
if limit := e.conn.MaxPayload(); int64(len(data)) > limit {
return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit)
}
ctx, cancel := context.WithTimeout(ctx, publishTimeout)
defer cancel()
msg := nats.NewMsg(e.subject)
msg.Data = data
reply, err := e.conn.RequestMsgWithContext(ctx, msg)
if err != nil {
return fmt.Errorf("publish spans: %w", err)
}
var ack struct {
Error *struct {
Description string `json:"description"`
} `json:"error"`
}
if err := json.Unmarshal(reply.Data, &ack); err != nil {
return fmt.Errorf("decode span ack: %w", err)
}
if ack.Error != nil {
return fmt.Errorf("publish spans: %s", ack.Error.Description)
}
return nil
}
// Relay forwards span batches from consumer to upstream until ctx ends. A
// batch that cannot be forwarded is returned to the stream and retried until
// the consumer's delivery limit drops it.
func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) {
if err := upstream.Start(ctx); err != nil {
log.ErrorContext(ctx, "span relay upstream start failed", "err", err)
return
}
defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }()
for ctx.Err() == nil {
msg, err := consumer.Fetch(ctx)
if errors.Is(err, eventbus.ErrNoMessage) {
continue
}
if err != nil {
if ctx.Err() == nil {
log.WarnContext(ctx, "span relay fetch failed", "err", err)
sleep(ctx, retryDelay)
}
continue
}
forward(ctx, msg, upstream, log)
}
}
func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) {
var req coltracepb.ExportTraceServiceRequest
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
_ = msg.Ack()
return
}
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
defer cancel()
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
log.WarnContext(ctx, "span relay upload failed", "err", err)
_ = msg.Nak()
sleep(ctx, retryDelay)
return
}
_ = msg.Ack()
}
func sleep(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}