195 lines
6.5 KiB
Go
195 lines
6.5 KiB
Go
// Package telemetry installs the process's OpenTelemetry tracer provider and
|
|
// builds the instrumentation options every RPC and HTTP edge shares.
|
|
package telemetry
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
|
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
|
|
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials"
|
|
|
|
"github.com/samcharles93/archie-core/internal/config"
|
|
)
|
|
|
|
const shutdownTimeout = 5 * time.Second
|
|
|
|
// Start installs the global tracer provider and propagators for component and
|
|
// returns the function that flushes and stops them. With no endpoint it
|
|
// installs nothing, so the global provider stays a no-op and the returned
|
|
// function does nothing. Call it before any server or client is built: the
|
|
// instrumentation reads the globals once, at construction.
|
|
func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) {
|
|
if cfg.Endpoint == "" {
|
|
return func() {}, nil
|
|
}
|
|
client, err := NewClient(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return StartWithClient(ctx, client, cfg.SampleRatio, component, version)
|
|
}
|
|
|
|
// NewClient builds the OTLP/gRPC trace client for cfg. It is not started.
|
|
func NewClient(cfg config.Telemetry) (otlptrace.Client, error) {
|
|
opts, err := exporterOptions(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return otlptracegrpc.NewClient(opts...), nil
|
|
}
|
|
|
|
// MaxExportBatch caps spans per export so one batch stays well under the
|
|
// 1 MiB NATS payload limit the container exporter publishes through.
|
|
const MaxExportBatch = 128
|
|
|
|
// Propagator is the trace-context propagator every process installs.
|
|
func Propagator() propagation.TextMapPropagator {
|
|
return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})
|
|
}
|
|
|
|
// StartWithClient installs the global tracer provider, exporting through
|
|
// client, and returns the function that flushes and stops it. A trace started
|
|
// by a caller keeps its caller's sampling decision; ratio applies to new ones.
|
|
func StartWithClient(ctx context.Context, client otlptrace.Client, ratio float64, component, version string) (func(), error) {
|
|
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
|
|
semconv.ServiceName(component),
|
|
semconv.ServiceVersion(version),
|
|
semconv.ServiceInstanceID(instanceID()),
|
|
))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("telemetry resource: %w", err)
|
|
}
|
|
exporter, err := otlptrace.New(ctx, client)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("telemetry exporter: %w", err)
|
|
}
|
|
tp := sdktrace.NewTracerProvider(
|
|
sdktrace.WithBatcher(exporter, sdktrace.WithMaxExportBatchSize(MaxExportBatch)),
|
|
sdktrace.WithResource(res),
|
|
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))),
|
|
)
|
|
otel.SetTracerProvider(tp)
|
|
otel.SetTextMapPropagator(Propagator())
|
|
return func() { shutdown(ctx, tp) }, nil
|
|
}
|
|
|
|
// shutdown flushes after ctx has typically ended, so the deadline hangs off
|
|
// a context detached from its cancellation.
|
|
func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) {
|
|
flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout)
|
|
defer cancel()
|
|
_ = tp.Shutdown(flush)
|
|
}
|
|
|
|
func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) {
|
|
opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)}
|
|
if len(cfg.Headers) > 0 {
|
|
opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers))
|
|
}
|
|
if cfg.Insecure {
|
|
return append(opts, otlptracegrpc.WithInsecure()), nil
|
|
}
|
|
if cfg.CAFile != "" {
|
|
pool, err := caPool(cfg.CAFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12})))
|
|
}
|
|
return opts, nil
|
|
}
|
|
|
|
func caPool(path string) (*x509.CertPool, error) {
|
|
pem, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read telemetry ca_file: %w", err)
|
|
}
|
|
pool := x509.NewCertPool()
|
|
if !pool.AppendCertsFromPEM(pem) {
|
|
return nil, errors.New("telemetry ca_file holds no certificate")
|
|
}
|
|
return pool, nil
|
|
}
|
|
|
|
func instanceID() string {
|
|
host, _ := os.Hostname()
|
|
return host + "-" + strconv.Itoa(os.Getpid())
|
|
}
|
|
|
|
// ServerOption traces every inbound RPC except health probes.
|
|
func ServerOption() grpc.ServerOption {
|
|
return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck()))))
|
|
}
|
|
|
|
// ClientOption traces every outbound RPC and sends the span context.
|
|
func ClientOption() grpc.DialOption {
|
|
return grpc.WithStatsHandler(otelgrpc.NewClientHandler())
|
|
}
|
|
|
|
// Handler traces every HTTP request except health probes, naming the span by
|
|
// the ServeMux pattern. Place it outermost.
|
|
func Handler(next http.Handler) http.Handler {
|
|
return otelhttp.NewHandler(next, "http",
|
|
otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }),
|
|
)
|
|
}
|
|
|
|
func isHealth(path string) bool {
|
|
return strings.HasPrefix(path, "/health")
|
|
}
|
|
|
|
// Transport traces outbound HTTP requests made through base.
|
|
func Transport(base http.RoundTripper) http.RoundTripper {
|
|
return otelhttp.NewTransport(base)
|
|
}
|
|
|
|
// Routed names the request's server span after the mux pattern that serves
|
|
// it. Middleware that copies the request hides the pattern from Handler, so
|
|
// the inner mux reports it itself.
|
|
func Routed(mux *http.ServeMux) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if _, pattern := mux.Handler(r); pattern != "" {
|
|
span := trace.SpanFromContext(r.Context())
|
|
span.SetName(spanName(r.Method, pattern))
|
|
span.SetAttributes(semconv.HTTPRoute(routeOf(pattern)))
|
|
}
|
|
mux.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// spanName is "METHOD /route"; a pattern that already names its method is
|
|
// used as is.
|
|
func spanName(method, pattern string) string {
|
|
if strings.Contains(pattern, " ") {
|
|
return pattern
|
|
}
|
|
return method + " " + pattern
|
|
}
|
|
|
|
func routeOf(pattern string) string {
|
|
if _, route, ok := strings.Cut(pattern, " "); ok {
|
|
return route
|
|
}
|
|
return pattern
|
|
}
|