// Package telemetry installs the process's OpenTelemetry tracer provider and // builds the instrumentation options every RPC and HTTP edge shares. package telemetry import ( "context" "crypto/tls" "crypto/x509" "errors" "fmt" "net/http" "os" "strconv" "strings" "time" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.43.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "github.com/samcharles93/archie-core/internal/config" ) const shutdownTimeout = 5 * time.Second // Start installs the global tracer provider and propagators for component and // returns the function that flushes and stops them. With no endpoint it // installs nothing, so the global provider stays a no-op and the returned // function does nothing. Call it before any server or client is built: the // instrumentation reads the globals once, at construction. func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) { if cfg.Endpoint == "" { return func() {}, nil } client, err := NewClient(cfg) if err != nil { return nil, err } return StartWithClient(ctx, client, cfg.SampleRatio, component, version) } // NewClient builds the OTLP/gRPC trace client for cfg. It is not started. func NewClient(cfg config.Telemetry) (otlptrace.Client, error) { opts, err := exporterOptions(cfg) if err != nil { return nil, err } return otlptracegrpc.NewClient(opts...), nil } // MaxExportBatch caps spans per export so one batch stays well under the // 1 MiB NATS payload limit the container exporter publishes through. const MaxExportBatch = 128 // Propagator is the trace-context propagator every process installs. func Propagator() propagation.TextMapPropagator { return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) } // StartWithClient installs the global tracer provider, exporting through // client, and returns the function that flushes and stops it. A trace started // by a caller keeps its caller's sampling decision; ratio applies to new ones. func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) { res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, semconv.ServiceName(component), semconv.ServiceVersion(version), semconv.ServiceInstanceID(instanceID()), )) if err != nil { return nil, fmt.Errorf("telemetry resource: %w", err) } exporter, err := otlptrace.New(ctx, client) if err != nil { return nil, fmt.Errorf("telemetry exporter: %w", err) } tp := sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)), sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))), ) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(Propagator()) return func() { shutdown(ctx, tp) }, nil } // shutdown flushes after ctx has typically ended, so the deadline hangs off // a context detached from its cancellation. func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) { flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout) defer cancel() _ = tp.Shutdown(flush) } func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) { opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)} if len(cfg.Headers) > 0 { opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers)) } if cfg.Insecure { return append(opts, otlptracegrpc.WithInsecure()), nil } if cfg.CAFile != "" { pool, err := caPool(cfg.CAFile) if err != nil { return nil, err } opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}))) } return opts, nil } func caPool(path string) (*x509.CertPool, error) { pem, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read telemetry ca_file: %w", err) } pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(pem) { return nil, errors.New("telemetry ca_file holds no certificate") } return pool, nil } func instanceID() string { host, _ := os.Hostname() return host + "-" + strconv.Itoa(os.Getpid()) } // ServerOption traces every inbound RPC except health probes. func ServerOption() grpc.ServerOption { return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck())))) } // ClientOption traces every outbound RPC and sends the span context. func ClientOption() grpc.DialOption { return grpc.WithStatsHandler(otelgrpc.NewClientHandler()) } // Handler traces every HTTP request except health probes, naming the span by // the ServeMux pattern. Place it outermost. func Handler(next http.Handler) http.Handler { return otelhttp.NewHandler(next, "http", otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }), ) } func isHealth(path string) bool { return strings.HasPrefix(path, "/health") } // Transport traces outbound HTTP requests made through base. func Transport(base http.RoundTripper) http.RoundTripper { return otelhttp.NewTransport(base) } // Routed names the request's server span after the mux pattern that serves // it. Middleware that copies the request hides the pattern from Handler, so // the inner mux reports it itself. func Routed(mux *http.ServeMux) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if _, pattern := mux.Handler(r); pattern != "" { span := trace.SpanFromContext(r.Context()) span.SetName(spanName(r.Method, pattern)) span.SetAttributes(semconv.HTTPRoute(routeOf(pattern))) } mux.ServeHTTP(w, r) }) } // spanName is "METHOD /route"; a pattern that already names its method is // used as is. func spanName(method, pattern string) string { if strings.Contains(pattern, " ") { return pattern } return method + " " + pattern } func routeOf(pattern string) string { if _, route, ok := strings.Cut(pattern, " "); ok { return route } return pattern }