// Package spanrelay carries spans from task containers to the collector. A // container has no route to the collector, so its tracer provider exports as // OTLP messages over JetStream and the daemon forwards them upstream. package spanrelay import ( "context" "encoding/json" "errors" "fmt" "log/slog" "strconv" "time" "github.com/nats-io/nats.go" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" tracepb "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" "github.com/samcharles93/archie-core/internal/eventbus" ) const ( // StreamName holds spans in flight. It is separate from ARCHIE_TASKS, // which is a work queue. StreamName = "ARCHIE_SPANS" // ConsumerName is the daemon's one durable consumer. ConsumerName = "archie-spans" // SubjectWildcard matches every task's span subject. SubjectWildcard = "archie.spans.>" // MaxAge bounds how long an undelivered batch is kept. MaxAge = time.Hour publishTimeout = 5 * time.Second uploadTimeout = 10 * time.Second retryDelay = 2 * time.Second ) // Subject is where a task's container publishes its span batches. func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) } // Exporter is an otlptrace.Client that publishes each batch to the span // stream and returns once the stream has acknowledged it. type Exporter struct { conn *nats.Conn subject string } var _ otlptrace.Client = (*Exporter)(nil) // NewExporter publishes taskID's spans over conn. func NewExporter(conn *nats.Conn, taskID int64) *Exporter { return &Exporter{conn: conn, subject: Subject(taskID)} } // Start is a no-op: the connection is already open. func (*Exporter) Start(context.Context) error { return nil } // Stop is a no-op: the connection belongs to the transport. func (*Exporter) Stop(context.Context) error { return nil } // UploadTraces publishes one batch and waits for the JetStream ack. func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error { data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans}) if err != nil { return fmt.Errorf("encode spans: %w", err) } if limit := e.conn.MaxPayload(); int64(len(data)) > limit { return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit) } ctx, cancel := context.WithTimeout(ctx, publishTimeout) defer cancel() msg := nats.NewMsg(e.subject) msg.Data = data reply, err := e.conn.RequestMsgWithContext(ctx, msg) if err != nil { return fmt.Errorf("publish spans: %w", err) } var ack struct { Error *struct { Description string `json:"description"` } `json:"error"` } if err := json.Unmarshal(reply.Data, &ack); err != nil { return fmt.Errorf("decode span ack: %w", err) } if ack.Error != nil { return fmt.Errorf("publish spans: %s", ack.Error.Description) } return nil } // Relay forwards span batches from consumer to upstream until ctx ends. A // batch that cannot be forwarded is returned to the stream and retried until // the consumer's delivery limit drops it. func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) { if err := upstream.Start(ctx); err != nil { log.ErrorContext(ctx, "span relay upstream start failed", "err", err) return } defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }() for ctx.Err() == nil { msg, err := consumer.Fetch(ctx) if errors.Is(err, eventbus.ErrNoMessage) { continue } if err != nil { if ctx.Err() == nil { log.WarnContext(ctx, "span relay fetch failed", "err", err) sleep(ctx, retryDelay) } continue } forward(ctx, msg, upstream, log) } } func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) { var req coltracepb.ExportTraceServiceRequest if err := proto.Unmarshal(msg.Data(), &req); err != nil { log.WarnContext(ctx, "span relay dropped undecodable batch", "subject", msg.Subject(), "err", err) _ = msg.Ack() return } upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout) defer cancel() if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil { log.WarnContext(ctx, "span relay upload failed", "err", err) _ = msg.Nak() sleep(ctx, retryDelay) return } _ = msg.Ack() } func sleep(ctx context.Context, d time.Duration) { t := time.NewTimer(d) defer t.Stop() select { case <-ctx.Done(): case <-t.C: } }