From 8524c3889299de820df0ee7c90067fadc8276cc6 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sun, 25 Oct 2026 23:17:34 +1100 Subject: [PATCH] feat(agent): export container spans over NATS through archied --- go.mod | 4 +- internal/app/archied/bootstrap.go | 6 + internal/app/archied/span_relay.go | 50 ++++++ internal/app/archied/task_broker.go | 2 + .../agenttransport/nats/tracing.go | 30 ++++ .../agenttransport/nats/transport.go | 9 ++ .../infrastructure/spanrelay/spanrelay.go | 144 ++++++++++++++++++ .../infrastructure/telemetry/telemetry.go | 42 ++++- 8 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 internal/app/archied/span_relay.go create mode 100644 internal/infrastructure/agenttransport/nats/tracing.go create mode 100644 internal/infrastructure/spanrelay/spanrelay.go diff --git a/go.mod b/go.mod index dfa422779..826f077cb 100644 --- a/go.mod +++ b/go.mod @@ -57,10 +57,8 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect go.opentelemetry.io/otel/metric v1.47.0 // indirect - go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect @@ -104,9 +102,11 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 go.opentelemetry.io/otel/sdk v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 + go.opentelemetry.io/proto/otlp v1.11.0 golang.org/x/crypto v0.58.0 golang.org/x/mod v0.42.0 golang.org/x/net v0.61.0 diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index 8ef9f4b13..de7a3e385 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -437,6 +437,12 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco return err } + if err := b.startSpanRelay(ctx, url, natsToken); err != nil { + log.Error("span relay failed", "err", err) + natsClient.Close() + reactionClient.Close() + return err + } b.natsClient = natsClient b.natsURL = url b.natsToken = natsToken diff --git a/internal/app/archied/span_relay.go b/internal/app/archied/span_relay.go new file mode 100644 index 000000000..4de254f3c --- /dev/null +++ b/internal/app/archied/span_relay.go @@ -0,0 +1,50 @@ +package archied + +import ( + "context" + "fmt" + + "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" + "github.com/samcharles93/archie-core/internal/infrastructure/spanrelay" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" +) + +// startSpanRelay forwards the spans task containers publish to the collector. +// It does nothing while telemetry is off, and the containers then export +// nothing. +func (b *boot) startSpanRelay(ctx context.Context, url, token string) error { + if b.cfg.Telemetry.Endpoint == "" { + return nil + } + maxAge := spanrelay.MaxAge + spans, err := nats.Connect(ctx, nats.Config{ + URL: url, + Token: token, + StreamName: spanrelay.StreamName, + ConsumerName: spanrelay.ConsumerName, + Subjects: []string{spanrelay.SubjectWildcard}, + FilterSubject: spanrelay.SubjectWildcard, + Retention: nats.FanOutRetention(), + MaxAge: &maxAge, + }, b.log) + if err != nil { + return fmt.Errorf("nats span stream connect: %w", err) + } + upstream, err := telemetry.NewClient(b.cfg.Telemetry) + if err != nil { + spans.Close() + return err + } + relayCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + spanrelay.Relay(relayCtx, spans, upstream, b.log) + }() + b.addCleanup(func() { + cancel() + <-done + spans.Close() + }) + return nil +} diff --git a/internal/app/archied/task_broker.go b/internal/app/archied/task_broker.go index c754015ae..b54c63ff8 100644 --- a/internal/app/archied/task_broker.go +++ b/internal/app/archied/task_broker.go @@ -9,6 +9,7 @@ import ( "github.com/samcharles93/archie-core/internal/forgerpc" agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats" "github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats" + "github.com/samcharles93/archie-core/internal/infrastructure/spanrelay" "github.com/samcharles93/archie-core/internal/worktreerpc" ) @@ -31,6 +32,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s publish := append([]string{ agentexec.SubjectForSystem(taskID), agentexec.SubjectForEvents(taskID), + spanrelay.Subject(taskID), worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush), }, forgerpc.Subjects(task.Identity)...) return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil diff --git a/internal/infrastructure/agenttransport/nats/tracing.go b/internal/infrastructure/agenttransport/nats/tracing.go new file mode 100644 index 000000000..f043514e4 --- /dev/null +++ b/internal/infrastructure/agenttransport/nats/tracing.go @@ -0,0 +1,30 @@ +package nats + +import ( + "context" + "log/slog" + + "github.com/samcharles93/archie-core/internal/buildinfo" + "github.com/samcharles93/archie-core/internal/infrastructure/spanrelay" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" + "github.com/samcharles93/archie-core/internal/taskrun" +) + +// startTracing installs the container's tracer provider on the first request +// that carries a trace context, which is how the daemon says tracing is on. +// Spans leave through the daemon, since the container has no route to a +// collector. +func (t *Transport) startTracing(ctx context.Context, request taskrun.Request, log *slog.Logger) { + if len(request.Trace) == 0 { + return + } + t.tracingOnce.Do(func() { + client := spanrelay.NewExporter(t.conn, request.Task.ID) + stop, err := telemetry.StartWithClient(ctx, client, 1, "agent", buildinfo.Version) + if err != nil { + log.Warn("container tracing disabled", "err", err) + return + } + t.stopTracing = stop + }) +} diff --git a/internal/infrastructure/agenttransport/nats/transport.go b/internal/infrastructure/agenttransport/nats/transport.go index 7f1fd0805..6b0e7c95e 100644 --- a/internal/infrastructure/agenttransport/nats/transport.go +++ b/internal/infrastructure/agenttransport/nats/transport.go @@ -9,6 +9,7 @@ import ( "log/slog" "strconv" "strings" + "sync" "time" natsio "github.com/nats-io/nats.go" @@ -60,6 +61,10 @@ type sdkSubscription interface { // worker has no JetStream consumer. type Transport struct { conn *natsio.Conn + // stopTracing flushes the container's spans; Close runs it before the + // connection goes. + tracingOnce sync.Once + stopTracing func() // state is the State Store client; closeState closes its connection. state *staterpc.Client closeState func() @@ -122,6 +127,9 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport, // Close releases all broker resources. func (t *Transport) Close() { + if t.stopTracing != nil { + t.stopTracing() + } t.conn.Close() if t.closeState != nil { t.closeState() @@ -315,6 +323,7 @@ func (t *Transport) handleTask( t.respondTask(msg, nil, fmt.Errorf("validate taskrun request: %w", err), log) return } + t.startTracing(ctx, request, log) ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(request.Trace)) ctx, span := natsrpc.StartServer(ctx, msg) defer span.End() diff --git a/internal/infrastructure/spanrelay/spanrelay.go b/internal/infrastructure/spanrelay/spanrelay.go new file mode 100644 index 000000000..d615a6080 --- /dev/null +++ b/internal/infrastructure/spanrelay/spanrelay.go @@ -0,0 +1,144 @@ +// Package spanrelay carries spans from task containers to the collector. A +// container has no route to the collector, so its tracer provider exports as +// OTLP messages over JetStream and the daemon forwards them upstream. +package spanrelay + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strconv" + "time" + + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" + tracepb "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/protobuf/proto" + + "github.com/samcharles93/archie-core/internal/eventbus" +) + +const ( + // StreamName holds spans in flight. It is separate from ARCHIE_TASKS, + // which is a work queue. + StreamName = "ARCHIE_SPANS" + // ConsumerName is the daemon's one durable consumer. + ConsumerName = "archie-spans" + // SubjectWildcard matches every task's span subject. + SubjectWildcard = "archie.spans.>" + // MaxAge bounds how long an undelivered batch is kept. + MaxAge = time.Hour + + publishTimeout = 5 * time.Second + uploadTimeout = 10 * time.Second + retryDelay = 2 * time.Second +) + +// Subject is where a task's container publishes its span batches. +func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) } + +// Exporter is an otlptrace.Client that publishes each batch to the span +// stream and returns once the stream has acknowledged it. +type Exporter struct { + conn *nats.Conn + subject string +} + +var _ otlptrace.Client = (*Exporter)(nil) + +// NewExporter publishes taskID's spans over conn. +func NewExporter(conn *nats.Conn, taskID int64) *Exporter { + return &Exporter{conn: conn, subject: Subject(taskID)} +} + +// Start is a no-op: the connection is already open. +func (*Exporter) Start(context.Context) error { return nil } + +// Stop is a no-op: the connection belongs to the transport. +func (*Exporter) Stop(context.Context) error { return nil } + +// UploadTraces publishes one batch and waits for the JetStream ack. +func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error { + data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans}) + if err != nil { + return fmt.Errorf("encode spans: %w", err) + } + if limit := e.conn.MaxPayload(); int64(len(data)) > limit { + return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit) + } + ctx, cancel := context.WithTimeout(ctx, publishTimeout) + defer cancel() + msg := nats.NewMsg(e.subject) + msg.Data = data + reply, err := e.conn.RequestMsgWithContext(ctx, msg) + if err != nil { + return fmt.Errorf("publish spans: %w", err) + } + var ack struct { + Error *struct { + Description string `json:"description"` + } `json:"error"` + } + if err := json.Unmarshal(reply.Data, &ack); err != nil { + return fmt.Errorf("decode span ack: %w", err) + } + if ack.Error != nil { + return fmt.Errorf("publish spans: %s", ack.Error.Description) + } + return nil +} + +// Relay forwards span batches from consumer to upstream until ctx ends. A +// batch that cannot be forwarded is returned to the stream and retried until +// the consumer's delivery limit drops it. +func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) { + if err := upstream.Start(ctx); err != nil { + log.Error("span relay upstream start failed", "err", err) + return + } + defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }() + for ctx.Err() == nil { + msg, err := consumer.Fetch(ctx) + if errors.Is(err, eventbus.ErrNoMessage) { + continue + } + if err != nil { + if ctx.Err() == nil { + log.Warn("span relay fetch failed", "err", err) + sleep(ctx, retryDelay) + } + continue + } + forward(ctx, msg, upstream, log) + } +} + +func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) { + var req coltracepb.ExportTraceServiceRequest + if err := proto.Unmarshal(msg.Data(), &req); err != nil { + log.Warn("span relay dropped undecodable batch", "subject", msg.Subject(), "err", err) + _ = msg.Ack() + return + } + upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout) + defer cancel() + if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil { + log.Warn("span relay upload failed", "err", err) + _ = msg.Nak() + sleep(ctx, retryDelay) + return + } + _ = msg.Ack() +} + +func sleep(ctx context.Context, d time.Duration) { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + case <-t.C: + } +} diff --git a/internal/infrastructure/telemetry/telemetry.go b/internal/infrastructure/telemetry/telemetry.go index c16aaf2d7..792449263 100644 --- a/internal/infrastructure/telemetry/telemetry.go +++ b/internal/infrastructure/telemetry/telemetry.go @@ -18,6 +18,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" @@ -40,6 +41,35 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string) if cfg.Endpoint == "" { return func() {}, nil } + client, err := NewClient(cfg) + if err != nil { + return nil, err + } + return StartWithClient(ctx, client, cfg.SampleRatio, component, version) +} + +// NewClient builds the OTLP/gRPC trace client for cfg. It is not started. +func NewClient(cfg config.Telemetry) (otlptrace.Client, error) { + opts, err := exporterOptions(cfg) + if err != nil { + return nil, err + } + return otlptracegrpc.NewClient(opts...), nil +} + +// MaxExportBatch caps spans per export so one batch stays well under the +// 1 MiB NATS payload limit the container exporter publishes through. +const MaxExportBatch = 128 + +// Propagator is the trace-context propagator every process installs. +func Propagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) +} + +// StartWithClient installs the global tracer provider, exporting through +// client, and returns the function that flushes and stops it. A trace started +// by a caller keeps its caller's sampling decision; ratio applies to new ones. +func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) { res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName(component), semconv.ServiceVersion(version), @@ -48,21 +78,17 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string) if err != nil { return nil, fmt.Errorf("telemetry resource: %w", err) } - opts, err := exporterOptions(cfg) - if err != nil { - return nil, err - } - exporter, err := otlptracegrpc.New(ctx, opts...) + exporter, err := otlptrace.New(ctx, client) if err != nil { return nil, fmt.Errorf("telemetry exporter: %w", err) } tp := sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), + sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)), sdktrace.WithResource(res), - sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))), ) otel.SetTracerProvider(tp) - otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + otel.SetTextMapPropagator(Propagator()) return func() { shutdown(ctx, tp) }, nil }