Files
archie/internal/infrastructure/telemetry/telemetry.go
T

195 lines
6.5 KiB
Go

// Package telemetry installs the process's OpenTelemetry tracer provider and
// builds the instrumentation options every RPC and HTTP edge shares.
package telemetry
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"github.com/samcharles93/archie-core/internal/config"
)
const shutdownTimeout = 5 * time.Second
// Start installs the global tracer provider and propagators for component and
// returns the function that flushes and stops them. With no endpoint it
// installs nothing, so the global provider stays a no-op and the returned
// function does nothing. Call it before any server or client is built: the
// instrumentation reads the globals once, at construction.
func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) {
if cfg.Endpoint == "" {
return func() {}, nil
}
client, err := NewClient(cfg)
if err != nil {
return nil, err
}
return StartWithClient(ctx, client, cfg.SampleRatio, component, version)
}
// NewClient builds the OTLP/gRPC trace client for cfg. It is not started.
func NewClient(cfg config.Telemetry) (otlptrace.Client, error) {
opts, err := exporterOptions(cfg)
if err != nil {
return nil, err
}
return otlptracegrpc.NewClient(opts...), nil
}
// MaxExportBatch caps spans per export so one batch stays well under the
// 1 MiB NATS payload limit the container exporter publishes through.
const MaxExportBatch = 128
// Propagator is the trace-context propagator every process installs.
func Propagator() propagation.TextMapPropagator {
return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
}
// StartWithClient installs the global tracer provider, exporting through
// client, and returns the function that flushes and stops it. A trace started
// by a caller keeps its caller's sampling decision; ratio applies to new ones.
func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) {
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
semconv.ServiceName(component),
semconv.ServiceVersion(version),
semconv.ServiceInstanceID(instanceID()),
))
if err != nil {
return nil, fmt.Errorf("telemetry resource: %w", err)
}
exporter, err := otlptrace.New(ctx, client)
if err != nil {
return nil, fmt.Errorf("telemetry exporter: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(Propagator())
return func() { shutdown(ctx, tp) }, nil
}
// shutdown flushes after ctx has typically ended, so the deadline hangs off
// a context detached from its cancellation.
func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) {
flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout)
defer cancel()
_ = tp.Shutdown(flush)
}
func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) {
opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)}
if len(cfg.Headers) > 0 {
opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers))
}
if cfg.Insecure {
return append(opts, otlptracegrpc.WithInsecure()), nil
}
if cfg.CAFile != "" {
pool, err := caPool(cfg.CAFile)
if err != nil {
return nil, err
}
opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12})))
}
return opts, nil
}
func caPool(path string) (*x509.CertPool, error) {
pem, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read telemetry ca_file: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, errors.New("telemetry ca_file holds no certificate")
}
return pool, nil
}
func instanceID() string {
host, _ := os.Hostname()
return host + "-" + strconv.Itoa(os.Getpid())
}
// ServerOption traces every inbound RPC except health probes.
func ServerOption() grpc.ServerOption {
return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck()))))
}
// ClientOption traces every outbound RPC and sends the span context.
func ClientOption() grpc.DialOption {
return grpc.WithStatsHandler(otelgrpc.NewClientHandler())
}
// Handler traces every HTTP request except health probes, naming the span by
// the ServeMux pattern. Place it outermost.
func Handler(next http.Handler) http.Handler {
return otelhttp.NewHandler(next, "http",
otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }),
)
}
func isHealth(path string) bool {
return strings.HasPrefix(path, "/health")
}
// Transport traces outbound HTTP requests made through base.
func Transport(base http.RoundTripper) http.RoundTripper {
return otelhttp.NewTransport(base)
}
// Routed names the request's server span after the mux pattern that serves
// it. Middleware that copies the request hides the pattern from Handler, so
// the inner mux reports it itself.
func Routed(mux *http.ServeMux) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if _, pattern := mux.Handler(r); pattern != "" {
span := trace.SpanFromContext(r.Context())
span.SetName(spanName(r.Method, pattern))
span.SetAttributes(semconv.HTTPRoute(routeOf(pattern)))
}
mux.ServeHTTP(w, r)
})
}
// spanName is "METHOD /route"; a pattern that already names its method is
// used as is.
func spanName(method, pattern string) string {
if strings.Contains(pattern, " ") {
return pattern
}
return method + " " + pattern
}
func routeOf(pattern string) string {
if _, route, ok := strings.Cut(pattern, " "); ok {
return route
}
return pattern
}