feat(agent): export container spans over NATS through archied

This commit is contained in:
sam committed 2026-10-25 23:17:34 +11:00
1 parent 4b7e843ead
commit 812f1bdd9e
8 files changed
+277 -10

No files matched your search

+2 -2
View File
@@ -57,10 +57,8 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect
@@ -104,9 +102,11 @@ require (
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
go.opentelemetry.io/proto/otlp v1.11.0
golang.org/x/crypto v0.58.0
golang.org/x/mod v0.42.0
golang.org/x/net v0.61.0
+6
View File
@@ -437,6 +437,12 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
return err
}
if err := b.startSpanRelay(ctx, url, natsToken); err != nil {
log.Error("span relay failed", "err", err)
natsClient.Close()
reactionClient.Close()
return err
}
b.natsClient = natsClient
b.natsURL = url
b.natsToken = natsToken
+50
View File
@@ -0,0 +1,50 @@
package archied
import (
"context"
"fmt"
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// startSpanRelay forwards the spans task containers publish to the collector.
// It does nothing while telemetry is off, and the containers then export
// nothing.
func (b *boot) startSpanRelay(ctx context.Context, url, token string) error {
if b.cfg.Telemetry.Endpoint == "" {
return nil
}
maxAge := spanrelay.MaxAge
spans, err := nats.Connect(ctx, nats.Config{
URL: url,
Token: token,
StreamName: spanrelay.StreamName,
ConsumerName: spanrelay.ConsumerName,
Subjects: []string{spanrelay.SubjectWildcard},
FilterSubject: spanrelay.SubjectWildcard,
Retention: nats.FanOutRetention(),
MaxAge: &maxAge,
}, b.log)
if err != nil {
return fmt.Errorf("nats span stream connect: %w", err)
}
upstream, err := telemetry.NewClient(b.cfg.Telemetry)
if err != nil {
spans.Close()
return err
}
relayCtx, cancel := context.WithCancel(ctx)
done := make(chan struct{})
go func() {
defer close(done)
spanrelay.Relay(relayCtx, spans, upstream, b.log)
}()
b.addCleanup(func() {
cancel()
<-done
spans.Close()
})
return nil
}
+2
View File
@@ -9,6 +9,7 @@ import (
"github.com/samcharles93/archie-core/internal/forgerpc"
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
"github.com/samcharles93/archie-core/internal/worktreerpc"
)
@@ -31,6 +32,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s
publish := append([]string{
agentexec.SubjectForSystem(taskID),
agentexec.SubjectForEvents(taskID),
spanrelay.Subject(taskID),
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
}, forgerpc.Subjects(task.Identity)...)
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
@@ -0,0 +1,30 @@
package nats
import (
"context"
"log/slog"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
"github.com/samcharles93/archie-core/internal/taskrun"
)
// startTracing installs the container's tracer provider on the first request
// that carries a trace context, which is how the daemon says tracing is on.
// Spans leave through the daemon, since the container has no route to a
// collector.
func (t *Transport) startTracing(ctx context.Context, request taskrun.Request, log *slog.Logger) {
if len(request.Trace) == 0 {
return
}
t.tracingOnce.Do(func() {
client := spanrelay.NewExporter(t.conn, request.Task.ID)
stop, err := telemetry.StartWithClient(ctx, client, 1, "agent", buildinfo.Version)
if err != nil {
log.Warn("container tracing disabled", "err", err)
return
}
t.stopTracing = stop
})
}
@@ -9,6 +9,7 @@ import (
"log/slog"
"strconv"
"strings"
"sync"
"time"
natsio "github.com/nats-io/nats.go"
@@ -60,6 +61,10 @@ type sdkSubscription interface {
// worker has no JetStream consumer.
type Transport struct {
conn *natsio.Conn
// stopTracing flushes the container's spans; Close runs it before the
// connection goes.
tracingOnce sync.Once
stopTracing func()
// state is the State Store client; closeState closes its connection.
state *staterpc.Client
closeState func()
@@ -122,6 +127,9 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
// Close releases all broker resources.
func (t *Transport) Close() {
if t.stopTracing != nil {
t.stopTracing()
}
t.conn.Close()
if t.closeState != nil {
t.closeState()
@@ -315,6 +323,7 @@ func (t *Transport) handleTask(
t.respondTask(msg, nil, fmt.Errorf("validate taskrun request: %w", err), log)
return
}
t.startTracing(ctx, request, log)
ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(request.Trace))
ctx, span := natsrpc.StartServer(ctx, msg)
defer span.End()
@@ -0,0 +1,144 @@
// Package spanrelay carries spans from task containers to the collector. A
// container has no route to the collector, so its tracer provider exports as
// OTLP messages over JetStream and the daemon forwards them upstream.
package spanrelay
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strconv"
"time"
"github.com/nats-io/nats.go"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
"google.golang.org/protobuf/proto"
"github.com/samcharles93/archie-core/internal/eventbus"
)
const (
// StreamName holds spans in flight. It is separate from ARCHIE_TASKS,
// which is a work queue.
StreamName = "ARCHIE_SPANS"
// ConsumerName is the daemon's one durable consumer.
ConsumerName = "archie-spans"
// SubjectWildcard matches every task's span subject.
SubjectWildcard = "archie.spans.>"
// MaxAge bounds how long an undelivered batch is kept.
MaxAge = time.Hour
publishTimeout = 5 * time.Second
uploadTimeout = 10 * time.Second
retryDelay = 2 * time.Second
)
// Subject is where a task's container publishes its span batches.
func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) }
// Exporter is an otlptrace.Client that publishes each batch to the span
// stream and returns once the stream has acknowledged it.
type Exporter struct {
conn *nats.Conn
subject string
}
var _ otlptrace.Client = (*Exporter)(nil)
// NewExporter publishes taskID's spans over conn.
func NewExporter(conn *nats.Conn, taskID int64) *Exporter {
return &Exporter{conn: conn, subject: Subject(taskID)}
}
// Start is a no-op: the connection is already open.
func (*Exporter) Start(context.Context) error { return nil }
// Stop is a no-op: the connection belongs to the transport.
func (*Exporter) Stop(context.Context) error { return nil }
// UploadTraces publishes one batch and waits for the JetStream ack.
func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans})
if err != nil {
return fmt.Errorf("encode spans: %w", err)
}
if limit := e.conn.MaxPayload(); int64(len(data)) > limit {
return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit)
}
ctx, cancel := context.WithTimeout(ctx, publishTimeout)
defer cancel()
msg := nats.NewMsg(e.subject)
msg.Data = data
reply, err := e.conn.RequestMsgWithContext(ctx, msg)
if err != nil {
return fmt.Errorf("publish spans: %w", err)
}
var ack struct {
Error *struct {
Description string `json:"description"`
} `json:"error"`
}
if err := json.Unmarshal(reply.Data, &ack); err != nil {
return fmt.Errorf("decode span ack: %w", err)
}
if ack.Error != nil {
return fmt.Errorf("publish spans: %s", ack.Error.Description)
}
return nil
}
// Relay forwards span batches from consumer to upstream until ctx ends. A
// batch that cannot be forwarded is returned to the stream and retried until
// the consumer's delivery limit drops it.
func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) {
if err := upstream.Start(ctx); err != nil {
log.Error("span relay upstream start failed", "err", err)
return
}
defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }()
for ctx.Err() == nil {
msg, err := consumer.Fetch(ctx)
if errors.Is(err, eventbus.ErrNoMessage) {
continue
}
if err != nil {
if ctx.Err() == nil {
log.Warn("span relay fetch failed", "err", err)
sleep(ctx, retryDelay)
}
continue
}
forward(ctx, msg, upstream, log)
}
}
func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) {
var req coltracepb.ExportTraceServiceRequest
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
log.Warn("span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
_ = msg.Ack()
return
}
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
defer cancel()
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
log.Warn("span relay upload failed", "err", err)
_ = msg.Nak()
sleep(ctx, retryDelay)
return
}
_ = msg.Ack()
}
func sleep(ctx context.Context, d time.Duration) {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
case <-t.C:
}
}
+34 -8
View File
@@ -18,6 +18,7 @@ import (
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
@@ -40,6 +41,35 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string)
if cfg.Endpoint == "" {
return func() {}, nil
}
client, err := NewClient(cfg)
if err != nil {
return nil, err
}
return StartWithClient(ctx, client, cfg.SampleRatio, component, version)
}
// NewClient builds the OTLP/gRPC trace client for cfg. It is not started.
func NewClient(cfg config.Telemetry) (otlptrace.Client, error) {
opts, err := exporterOptions(cfg)
if err != nil {
return nil, err
}
return otlptracegrpc.NewClient(opts...), nil
}
// MaxExportBatch caps spans per export so one batch stays well under the
// 1 MiB NATS payload limit the container exporter publishes through.
const MaxExportBatch = 128
// Propagator is the trace-context propagator every process installs.
func Propagator() propagation.TextMapPropagator {
return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
}
// StartWithClient installs the global tracer provider, exporting through
// client, and returns the function that flushes and stops it. A trace started
// by a caller keeps its caller's sampling decision; ratio applies to new ones.
func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) {
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
semconv.ServiceName(component),
semconv.ServiceVersion(version),
@@ -48,21 +78,17 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string)
if err != nil {
return nil, fmt.Errorf("telemetry resource: %w", err)
}
opts, err := exporterOptions(cfg)
if err != nil {
return nil, err
}
exporter, err := otlptracegrpc.New(ctx, opts...)
exporter, err := otlptrace.New(ctx, client)
if err != nil {
return nil, fmt.Errorf("telemetry exporter: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
otel.SetTextMapPropagator(Propagator())
return func() { shutdown(ctx, tp) }, nil
}