feat(agent): export container spans over NATS through archied
This commit is contained in:
8 files changed
+277
-10
No files matched your search
@@ -57,10 +57,8 @@ require (
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect
|
||||
@@ -104,9 +102,11 @@ require (
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.opentelemetry.io/otel v1.47.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
|
||||
go.opentelemetry.io/otel/sdk v1.47.0
|
||||
go.opentelemetry.io/otel/trace v1.47.0
|
||||
go.opentelemetry.io/proto/otlp v1.11.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/mod v0.42.0
|
||||
golang.org/x/net v0.61.0
|
||||
|
||||
@@ -437,6 +437,12 @@ func (b *boot) connectNATS(ctx context.Context) error { // embedded broker disco
|
||||
return err
|
||||
}
|
||||
|
||||
if err := b.startSpanRelay(ctx, url, natsToken); err != nil {
|
||||
log.Error("span relay failed", "err", err)
|
||||
natsClient.Close()
|
||||
reactionClient.Close()
|
||||
return err
|
||||
}
|
||||
b.natsClient = natsClient
|
||||
b.natsURL = url
|
||||
b.natsToken = natsToken
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package archied
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// startSpanRelay forwards the spans task containers publish to the collector.
|
||||
// It does nothing while telemetry is off, and the containers then export
|
||||
// nothing.
|
||||
func (b *boot) startSpanRelay(ctx context.Context, url, token string) error {
|
||||
if b.cfg.Telemetry.Endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
maxAge := spanrelay.MaxAge
|
||||
spans, err := nats.Connect(ctx, nats.Config{
|
||||
URL: url,
|
||||
Token: token,
|
||||
StreamName: spanrelay.StreamName,
|
||||
ConsumerName: spanrelay.ConsumerName,
|
||||
Subjects: []string{spanrelay.SubjectWildcard},
|
||||
FilterSubject: spanrelay.SubjectWildcard,
|
||||
Retention: nats.FanOutRetention(),
|
||||
MaxAge: &maxAge,
|
||||
}, b.log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("nats span stream connect: %w", err)
|
||||
}
|
||||
upstream, err := telemetry.NewClient(b.cfg.Telemetry)
|
||||
if err != nil {
|
||||
spans.Close()
|
||||
return err
|
||||
}
|
||||
relayCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
spanrelay.Relay(relayCtx, spans, upstream, b.log)
|
||||
}()
|
||||
b.addCleanup(func() {
|
||||
cancel()
|
||||
<-done
|
||||
spans.Close()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/forgerpc"
|
||||
agentnats "github.com/samcharles93/archie-core/internal/infrastructure/agenttransport/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/eventbus/nats"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
|
||||
"github.com/samcharles93/archie-core/internal/worktreerpc"
|
||||
)
|
||||
|
||||
@@ -31,6 +32,7 @@ func (b taskBroker) TaskSubjects(ctx context.Context, taskID int64, credential s
|
||||
publish := append([]string{
|
||||
agentexec.SubjectForSystem(taskID),
|
||||
agentexec.SubjectForEvents(taskID),
|
||||
spanrelay.Subject(taskID),
|
||||
worktreerpc.SubjectFor(task.Identity, worktreerpc.SubjectPush),
|
||||
}, forgerpc.Subjects(task.Identity)...)
|
||||
return nats.TaskSubjects{Publish: publish, Subscribe: []string{agentnats.SubjectForTask(taskID)}}, nil
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package nats
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/spanrelay"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
"github.com/samcharles93/archie-core/internal/taskrun"
|
||||
)
|
||||
|
||||
// startTracing installs the container's tracer provider on the first request
|
||||
// that carries a trace context, which is how the daemon says tracing is on.
|
||||
// Spans leave through the daemon, since the container has no route to a
|
||||
// collector.
|
||||
func (t *Transport) startTracing(ctx context.Context, request taskrun.Request, log *slog.Logger) {
|
||||
if len(request.Trace) == 0 {
|
||||
return
|
||||
}
|
||||
t.tracingOnce.Do(func() {
|
||||
client := spanrelay.NewExporter(t.conn, request.Task.ID)
|
||||
stop, err := telemetry.StartWithClient(ctx, client, 1, "agent", buildinfo.Version)
|
||||
if err != nil {
|
||||
log.Warn("container tracing disabled", "err", err)
|
||||
return
|
||||
}
|
||||
t.stopTracing = stop
|
||||
})
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
@@ -60,6 +61,10 @@ type sdkSubscription interface {
|
||||
// worker has no JetStream consumer.
|
||||
type Transport struct {
|
||||
conn *natsio.Conn
|
||||
// stopTracing flushes the container's spans; Close runs it before the
|
||||
// connection goes.
|
||||
tracingOnce sync.Once
|
||||
stopTracing func()
|
||||
// state is the State Store client; closeState closes its connection.
|
||||
state *staterpc.Client
|
||||
closeState func()
|
||||
@@ -122,6 +127,9 @@ func Connect(ctx context.Context, config Config, log *slog.Logger) (*Transport,
|
||||
|
||||
// Close releases all broker resources.
|
||||
func (t *Transport) Close() {
|
||||
if t.stopTracing != nil {
|
||||
t.stopTracing()
|
||||
}
|
||||
t.conn.Close()
|
||||
if t.closeState != nil {
|
||||
t.closeState()
|
||||
@@ -315,6 +323,7 @@ func (t *Transport) handleTask(
|
||||
t.respondTask(msg, nil, fmt.Errorf("validate taskrun request: %w", err), log)
|
||||
return
|
||||
}
|
||||
t.startTracing(ctx, request, log)
|
||||
ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(request.Trace))
|
||||
ctx, span := natsrpc.StartServer(ctx, msg)
|
||||
defer span.End()
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package spanrelay carries spans from task containers to the collector. A
|
||||
// container has no route to the collector, so its tracer provider exports as
|
||||
// OTLP messages over JetStream and the daemon forwards them upstream.
|
||||
package spanrelay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
||||
coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
|
||||
tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/eventbus"
|
||||
)
|
||||
|
||||
const (
|
||||
// StreamName holds spans in flight. It is separate from ARCHIE_TASKS,
|
||||
// which is a work queue.
|
||||
StreamName = "ARCHIE_SPANS"
|
||||
// ConsumerName is the daemon's one durable consumer.
|
||||
ConsumerName = "archie-spans"
|
||||
// SubjectWildcard matches every task's span subject.
|
||||
SubjectWildcard = "archie.spans.>"
|
||||
// MaxAge bounds how long an undelivered batch is kept.
|
||||
MaxAge = time.Hour
|
||||
|
||||
publishTimeout = 5 * time.Second
|
||||
uploadTimeout = 10 * time.Second
|
||||
retryDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
// Subject is where a task's container publishes its span batches.
|
||||
func Subject(taskID int64) string { return "archie.spans." + strconv.FormatInt(taskID, 10) }
|
||||
|
||||
// Exporter is an otlptrace.Client that publishes each batch to the span
|
||||
// stream and returns once the stream has acknowledged it.
|
||||
type Exporter struct {
|
||||
conn *nats.Conn
|
||||
subject string
|
||||
}
|
||||
|
||||
var _ otlptrace.Client = (*Exporter)(nil)
|
||||
|
||||
// NewExporter publishes taskID's spans over conn.
|
||||
func NewExporter(conn *nats.Conn, taskID int64) *Exporter {
|
||||
return &Exporter{conn: conn, subject: Subject(taskID)}
|
||||
}
|
||||
|
||||
// Start is a no-op: the connection is already open.
|
||||
func (*Exporter) Start(context.Context) error { return nil }
|
||||
|
||||
// Stop is a no-op: the connection belongs to the transport.
|
||||
func (*Exporter) Stop(context.Context) error { return nil }
|
||||
|
||||
// UploadTraces publishes one batch and waits for the JetStream ack.
|
||||
func (e *Exporter) UploadTraces(ctx context.Context, spans []*tracepb.ResourceSpans) error {
|
||||
data, err := proto.Marshal(&coltracepb.ExportTraceServiceRequest{ResourceSpans: spans})
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode spans: %w", err)
|
||||
}
|
||||
if limit := e.conn.MaxPayload(); int64(len(data)) > limit {
|
||||
return fmt.Errorf("span batch of %d bytes exceeds the %d byte payload limit", len(data), limit)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, publishTimeout)
|
||||
defer cancel()
|
||||
msg := nats.NewMsg(e.subject)
|
||||
msg.Data = data
|
||||
reply, err := e.conn.RequestMsgWithContext(ctx, msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish spans: %w", err)
|
||||
}
|
||||
var ack struct {
|
||||
Error *struct {
|
||||
Description string `json:"description"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(reply.Data, &ack); err != nil {
|
||||
return fmt.Errorf("decode span ack: %w", err)
|
||||
}
|
||||
if ack.Error != nil {
|
||||
return fmt.Errorf("publish spans: %s", ack.Error.Description)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Relay forwards span batches from consumer to upstream until ctx ends. A
|
||||
// batch that cannot be forwarded is returned to the stream and retried until
|
||||
// the consumer's delivery limit drops it.
|
||||
func Relay(ctx context.Context, consumer eventbus.Consumer, upstream otlptrace.Client, log *slog.Logger) {
|
||||
if err := upstream.Start(ctx); err != nil {
|
||||
log.Error("span relay upstream start failed", "err", err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = upstream.Stop(context.WithoutCancel(ctx)) }()
|
||||
for ctx.Err() == nil {
|
||||
msg, err := consumer.Fetch(ctx)
|
||||
if errors.Is(err, eventbus.ErrNoMessage) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() == nil {
|
||||
log.Warn("span relay fetch failed", "err", err)
|
||||
sleep(ctx, retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
forward(ctx, msg, upstream, log)
|
||||
}
|
||||
}
|
||||
|
||||
func forward(ctx context.Context, msg eventbus.Message, upstream otlptrace.Client, log *slog.Logger) {
|
||||
var req coltracepb.ExportTraceServiceRequest
|
||||
if err := proto.Unmarshal(msg.Data(), &req); err != nil {
|
||||
log.Warn("span relay dropped undecodable batch", "subject", msg.Subject(), "err", err)
|
||||
_ = msg.Ack()
|
||||
return
|
||||
}
|
||||
upload, cancel := context.WithTimeout(context.WithoutCancel(ctx), uploadTimeout)
|
||||
defer cancel()
|
||||
if err := upstream.UploadTraces(upload, req.GetResourceSpans()); err != nil {
|
||||
log.Warn("span relay upload failed", "err", err)
|
||||
_ = msg.Nak()
|
||||
sleep(ctx, retryDelay)
|
||||
return
|
||||
}
|
||||
_ = msg.Ack()
|
||||
}
|
||||
|
||||
func sleep(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
@@ -40,6 +41,35 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string)
|
||||
if cfg.Endpoint == "" {
|
||||
return func() {}, nil
|
||||
}
|
||||
client, err := NewClient(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return StartWithClient(ctx, client, cfg.SampleRatio, component, version)
|
||||
}
|
||||
|
||||
// NewClient builds the OTLP/gRPC trace client for cfg. It is not started.
|
||||
func NewClient(cfg config.Telemetry) (otlptrace.Client, error) {
|
||||
opts, err := exporterOptions(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return otlptracegrpc.NewClient(opts...), nil
|
||||
}
|
||||
|
||||
// MaxExportBatch caps spans per export so one batch stays well under the
|
||||
// 1 MiB NATS payload limit the container exporter publishes through.
|
||||
const MaxExportBatch = 128
|
||||
|
||||
// Propagator is the trace-context propagator every process installs.
|
||||
func Propagator() propagation.TextMapPropagator {
|
||||
return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
|
||||
}
|
||||
|
||||
// StartWithClient installs the global tracer provider, exporting through
|
||||
// client, and returns the function that flushes and stops it. A trace started
|
||||
// by a caller keeps its caller's sampling decision; ratio applies to new ones.
|
||||
func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) {
|
||||
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
|
||||
semconv.ServiceName(component),
|
||||
semconv.ServiceVersion(version),
|
||||
@@ -48,21 +78,17 @@ func Start(ctx context.Context, cfg config.Telemetry, component, version string)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telemetry resource: %w", err)
|
||||
}
|
||||
opts, err := exporterOptions(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exporter, err := otlptracegrpc.New(ctx, opts...)
|
||||
exporter, err := otlptrace.New(ctx, client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telemetry exporter: %w", err)
|
||||
}
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(exporter),
|
||||
sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
|
||||
otel.SetTextMapPropagator(Propagator())
|
||||
return func() { shutdown(ctx, tp) }, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user