diff --git a/config.example.toml b/config.example.toml index f22e56a85..63d9fe8eb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -45,6 +45,14 @@ task_wall_clock = "4h" # a whole task run listen = "127.0.0.1:8485" # dependency_timeout = "5s" +# Trace export. Without an endpoint nothing is exported. +# [telemetry] +# endpoint = "127.0.0.1:4317" # OTLP/gRPC collector +# insecure = false # plaintext to the collector +# ca_file = "" # PEM bundle verifying the collector +# sample_ratio = 1.0 # share of new traces recorded +# headers = { authorization = "Bearer ..." } + [web] listen = "127.0.0.1:8484" # "off" disables the dashboard # trust_forwarded_headers = false # only behind a proxy you control diff --git a/go.mod b/go.mod index 270501eb8..dfa422779 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/ProtonMail/go-crypto v1.5.2 // indirect github.com/antithesishq/antithesis-sdk-go v0.8.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.5 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -28,6 +29,7 @@ require ( github.com/go-webauthn/x v0.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-tpm v0.9.8 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/yamux v0.1.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -55,9 +57,10 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect go.opentelemetry.io/otel/log v1.47.0 // indirect go.opentelemetry.io/otel/metric v1.47.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect @@ -98,7 +101,11 @@ require ( go.lsp.dev/jsonrpc2 v1.0.1 go.lsp.dev/protocol v1.0.1 go.lsp.dev/uri v1.0.1 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/crypto v0.58.0 golang.org/x/mod v0.42.0 diff --git a/go.sum b/go.sum index 1550dd7c1..865512801 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/ github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/cedar-policy/cedar-go v1.8.0 h1:9gcU7EHXwHC2RMdpph68yTAkdB3behTTssC+kt4GoS8= github.com/cedar-policy/cedar-go v1.8.0/go.mod h1:h5+3CVW1oI5LXVskJG+my9TFCYI5yjh/+Ul3EJie6MI= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= @@ -96,6 +98,8 @@ github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+ github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs= @@ -207,10 +211,16 @@ go.lsp.dev/uri v1.0.1 h1:ywxJN1UXoQ+v0I9lkoQNMmsm7lub5G9ZG7KXT8YTx30= go.lsp.dev/uri v1.0.1/go.mod h1:06a0ghafs4PuqkLsJGUmlgPHw6+kH79edRXjOT/rA8s= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw= go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc= go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= @@ -221,6 +231,10 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index ab8e87a6d..8ef9f4b13 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -223,7 +223,7 @@ func (b *boot) cleanup() { } } -func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error { +func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) error { loader, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath) if err != nil { return err @@ -233,20 +233,23 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error b.cfg = b.doc.Config b.cfgHolder = config.NewHolder(b.cfg) b.currentProvenance.Store(&b.doc.Provenance) - b.setupLogging() - return nil + return b.setupLogging(ctx) } // setupLogging stays on stderr for an offline command (stderrLog): creating, // appending to or rotating the deployment's log is a side effect a diagnosis // must not have. -func (b *boot) setupLogging() { +func (b *boot) setupLogging(ctx context.Context) error { if b.stderrLog { - return + return nil + } + logs, err := servicekit.Logging(ctx, b.cfg, "daemon") + if err != nil { + return err } - logs := servicekit.Logging(b.cfg, "daemon") b.log, b.logFeed, b.taskLogs = logs.Log, logs.Feed, logs.TaskLogs b.addCleanup(func() { _ = logs.Closer.Close() }) + return nil } // openStores wires the secret registry and the conversation store. The diff --git a/internal/app/archiegateway/server.go b/internal/app/archiegateway/server.go index 2688d1a68..00899b468 100644 --- a/internal/app/archiegateway/server.go +++ b/internal/app/archiegateway/server.go @@ -109,7 +109,7 @@ type server struct { func Run(ctx context.Context, options Options) error { b := &server{log: slog.New(slog.NewJSONHandler(os.Stderr, nil))} defer b.cleanup() - if err := b.loadConfig(options.Config, options.Overlay); err != nil { + if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { return err } if err := b.openState(ctx); err != nil { @@ -151,7 +151,7 @@ func (b *server) chatLLM() *runtime.Runtime { return b.llm.Load() } func (b *server) setLLM(rt *runtime.Runtime) { b.llm.Store(rt) } func (b *server) announceReady() { sdnotify.Ready(b.log) } -func (b *server) loadConfig(cfgPath, overlayPath string) error { +func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error { _, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath) if err != nil { return err @@ -160,7 +160,10 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error { b.cfgPath = cfgPath b.catalog = modelcatalog.NewCatalog(cfgPath) b.cfgHolder = config.NewHolder(b.cfg) - logs := servicekit.Logging(b.cfg, "gateway") + logs, err := servicekit.Logging(ctx, b.cfg, "gateway") + if err != nil { + return err + } b.log, b.logFeed = logs.Log, logs.Feed b.addCleanup(func() { _ = logs.Closer.Close() }) b.secrets = secret.NewRegistry() diff --git a/internal/app/archiemessaging/config.go b/internal/app/archiemessaging/config.go index 3c926de59..f29df79c7 100644 --- a/internal/app/archiemessaging/config.go +++ b/internal/app/archiemessaging/config.go @@ -26,6 +26,7 @@ type ResolvedConfig struct { WorkDir string BotUser string ShowToolCalls bool + Telemetry config.Telemetry } type projection struct { @@ -37,6 +38,7 @@ type projection struct { workDir string botUser string showToolCalls bool + telemetry config.Telemetry } // Resolve loads the configuration, extracts the messaging projection and @@ -71,6 +73,7 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) { WorkDir: proj.workDir, BotUser: proj.botUser, ShowToolCalls: proj.showToolCalls, + Telemetry: proj.telemetry, }, nil } @@ -95,6 +98,7 @@ func project(cfg config.Config) projection { workDir: cfg.WorkDir, botUser: cfg.BotUser, showToolCalls: cfg.Chat.ShowToolCalls, + telemetry: cfg.Telemetry, } } diff --git a/internal/app/archiemessaging/run.go b/internal/app/archiemessaging/run.go index 0ad0f3277..e069f4d77 100644 --- a/internal/app/archiemessaging/run.go +++ b/internal/app/archiemessaging/run.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc" "github.com/samcharles93/archie-core/internal/app/servicekit" + "github.com/samcharles93/archie-core/internal/buildinfo" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/applystatus" "github.com/samcharles93/archie-core/internal/domain/messaging" @@ -22,20 +23,30 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" "github.com/samcharles93/archie-core/internal/infrastructure/secretengine" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" "github.com/samcharles93/archie-core/internal/logging" "github.com/samcharles93/archie-core/internal/releaseupdate" "github.com/samcharles93/archie-core/internal/secret" ) -// Run runs the standalone Messaging Service process until ctx is cancelled. -func Run(ctx context.Context, o Options) error { - log := slog.Default().With("service", "archie-messaging") - - log, feed := servicekit.Diagnostics(log, presence.Messaging) +// resolveTraced resolves the configuration and starts trace export from it. +func resolveTraced(ctx context.Context, o Options, log *slog.Logger) (ResolvedConfig, func(), error) { cfg, err := Resolve(o, log) if err != nil { - return fmt.Errorf("resolve messaging options: %w", err) + return ResolvedConfig{}, nil, fmt.Errorf("resolve messaging options: %w", err) } + stop, err := telemetry.Start(ctx, cfg.Telemetry, "messaging", buildinfo.Version) + return cfg, stop, err +} + +// Run runs the standalone Messaging Service process until ctx is cancelled. +func Run(ctx context.Context, o Options) error { + log, feed := servicekit.Diagnostics(slog.Default().With("service", "archie-messaging"), presence.Messaging) + cfg, stopTracing, err := resolveTraced(ctx, o, log) + if err != nil { + return err + } + defer stopTracing() chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Identity) if err != nil { diff --git a/internal/app/archieui/config.go b/internal/app/archieui/config.go index 878d2bb81..1aa9d3d15 100644 --- a/internal/app/archieui/config.go +++ b/internal/app/archieui/config.go @@ -27,6 +27,7 @@ func Resolve(o Options, log *slog.Logger) (Options, error) { o = merge(o, cfg) tls = cfg.tls o.instance = cfg.instance + o.telemetry = cfg.telemetry } } if o.instance == "" { @@ -70,6 +71,7 @@ type projection struct { tls config.TLS capture CaptureOptions instance string + telemetry config.Telemetry } func project(cfg config.Config) projection { @@ -82,6 +84,7 @@ func project(cfg config.Config) projection { daemon: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameDaemon).Target}, tls: cfg.TLS, instance: cfg.Instance, + telemetry: cfg.Telemetry, capture: CaptureOptions{ Retention: cfg.Capture.Retention.Std(), MaxEvents: cfg.Capture.MaxEvents, diff --git a/internal/app/archieui/options.go b/internal/app/archieui/options.go index 582b169bb..c3f80535f 100644 --- a/internal/app/archieui/options.go +++ b/internal/app/archieui/options.go @@ -5,6 +5,7 @@ package archieui import ( "time" + "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" ) @@ -78,6 +79,8 @@ type Options struct { // instance is the deployment name, from the configuration source or // ARCHIE_INSTANCE. instance string + // telemetry is the [telemetry] section, read only by Run. + telemetry config.Telemetry } // CaptureOptions carries the [capture] settings the capture receiver needs. diff --git a/internal/app/archieui/run.go b/internal/app/archieui/run.go index 2f23479cf..322e4007a 100644 --- a/internal/app/archieui/run.go +++ b/internal/app/archieui/run.go @@ -14,6 +14,7 @@ import ( "time" "github.com/samcharles93/archie-core/internal/app/servicekit" + "github.com/samcharles93/archie-core/internal/buildinfo" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/presence" infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access" @@ -23,6 +24,7 @@ import ( "github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc" "github.com/samcharles93/archie-core/internal/infrastructure/harnessrpc" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" "github.com/samcharles93/archie-core/internal/webui" ) @@ -34,6 +36,16 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log return infraaccess.StartLive(ctx, store.ListPolicies, store.PoliciesVersion, log) } +// resolveTraced resolves the options and starts trace export from them. +func resolveTraced(ctx context.Context, options Options, log *slog.Logger) (Options, func(), error) { + opts, err := Resolve(options, log) + if err != nil { + return Options{}, nil, err + } + stop, err := telemetry.Start(ctx, opts.telemetry, "ui", buildinfo.Version) + return opts, stop, err +} + // Run serves the dashboard until ctx is cancelled. It resolves its own // options, dials the Gateway and State Store contracts itself, composes the // HTTP surface over them, and unwinds listener then clients in reverse order @@ -41,10 +53,11 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log func Run(ctx context.Context, options Options) error { log := slog.New(slog.NewJSONHandler(os.Stderr, nil)).With("component", "ui") log, feed := servicekit.Diagnostics(log, presence.UI) - opts, err := Resolve(options, log) + opts, stopTracing, err := resolveTraced(ctx, options, log) if err != nil { return err } + defer stopTracing() warnPlainHTTP(log, opts) var cleanups []func() defer func() { @@ -157,7 +170,7 @@ func wireHarness(opts Options, srv *webui.Server, cleanups *[]func()) error { // serve runs handler until ctx ends, then shuts down within // ShutdownTimeout. func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error { - server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout} + server := &http.Server{Handler: telemetry.Handler(handler), ReadHeaderTimeout: opts.ReadHeaderTimeout} serveErr := make(chan error, 1) go func() { serveErr <- server.Serve(listener) }() select { diff --git a/internal/app/servicekit/config.go b/internal/app/servicekit/config.go index 1ea09433d..e69bb09fa 100644 --- a/internal/app/servicekit/config.go +++ b/internal/app/servicekit/config.go @@ -1,14 +1,17 @@ package servicekit import ( + "context" "fmt" "io" "log/slog" "os" "path/filepath" + "github.com/samcharles93/archie-core/internal/buildinfo" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/infrastructure/configuration" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" "github.com/samcharles93/archie-core/internal/logging" ) @@ -37,12 +40,18 @@ type Logs struct { Closer io.Closer } -// Logging opens the configured log for component. A file log that cannot be -// opened is reported and logging continues on stderr. -func Logging(cfg config.Config, component string) Logs { +// Logging opens the configured log for component and starts trace export. A +// file log that cannot be opened is reported and logging continues on stderr; +// a trace exporter that cannot be built fails startup. Closing flushes the +// exporter, then closes the log, so shutdown records are still written. +func Logging(ctx context.Context, cfg config.Config, component string) (Logs, error) { + stopTracing, err := telemetry.Start(ctx, cfg.Telemetry, component, buildinfo.Version) + if err != nil { + return Logs{}, err + } feed := logging.NewFeed(1000) taskLogs := logging.NewTaskRegistry(filepath.Join(cfg.StateDir, "logs", "tasks"), feed, logging.TaskSinkOptions{}) - fileLog, closer, err := logging.New(logging.Options{ + fileLog, fileCloser, err := logging.New(logging.Options{ File: cfg.Log.File, MaxSizeMB: cfg.Log.MaxSizeMB, Keep: cfg.Log.Keep, @@ -56,9 +65,17 @@ func Logging(cfg config.Config, component string) Logs { } else if cfg.Log.File != "" { log.Info("logging to file", "path", cfg.Log.File) } - return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer} + closer := closerFunc(func() error { + stopTracing() + return fileCloser.Close() + }) + return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer}, nil } +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + // Diagnostics captures a service's stderr log for its recent-log contract. func Diagnostics(log *slog.Logger, service string) (*slog.Logger, *logging.Feed) { feed := logging.NewFeed(1000) diff --git a/internal/app/servicekit/grpcserver.go b/internal/app/servicekit/grpcserver.go index c2c9706da..966e2712a 100644 --- a/internal/app/servicekit/grpcserver.go +++ b/internal/app/servicekit/grpcserver.go @@ -7,6 +7,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/keepalive" + + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) // maxConcurrentStreams caps the calls one connection holds open at once, so a @@ -20,6 +22,7 @@ const maxConcurrentStreams = 256 // the OS gives up on the socket. func ServerLimits() []grpc.ServerOption { return []grpc.ServerOption{ + telemetry.ServerOption(), grpc.MaxConcurrentStreams(maxConcurrentStreams), grpc.ConnectionTimeout(10 * time.Second), grpc.KeepaliveParams(keepalive.ServerParameters{ diff --git a/internal/app/statestore/boot.go b/internal/app/statestore/boot.go index 4edb6c28c..c464a748a 100644 --- a/internal/app/statestore/boot.go +++ b/internal/app/statestore/boot.go @@ -48,7 +48,7 @@ func (b *server) cleanup() { } } -func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) error { +func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error { _, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath) if err != nil { return err @@ -57,7 +57,10 @@ func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) erro if b.stderrLog { return nil } - logs := servicekit.Logging(b.cfg, "state-store") + logs, err := servicekit.Logging(ctx, b.cfg, "state-store") + if err != nil { + return err + } b.log, b.taskLogs, b.logFeed = logs.Log, logs.TaskLogs, logs.Feed b.addCleanup(func() { _ = logs.Closer.Close() }) return nil diff --git a/internal/config/config.go b/internal/config/config.go index c1b994b97..ecbf3e574 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -408,6 +408,7 @@ type Config struct { Web Web `toml:"web" yaml:"web"` Health Health `toml:"health" yaml:"health"` Log Log `toml:"log" yaml:"log"` + Telemetry Telemetry `toml:"telemetry" yaml:"telemetry"` Notify Notify `toml:"notify" yaml:"notify"` NATS NATSConfig `toml:"nats" yaml:"nats"` Containers ContainerConfig `toml:"containers" yaml:"containers"` @@ -884,6 +885,23 @@ type Log struct { Quiet bool `toml:"quiet" yaml:"quiet"` } +// Telemetry configures OTLP trace export. An empty Endpoint disables tracing: +// nothing is exported and no tracer provider is installed. +type Telemetry struct { + // Endpoint is the OTLP/gRPC collector address, host:port. + Endpoint string `toml:"endpoint" yaml:"endpoint"` + // Headers are sent with every export, for collector authentication. + Headers map[string]string `toml:"headers" yaml:"headers"` + // Insecure exports over plaintext gRPC instead of TLS. + Insecure bool `toml:"insecure" yaml:"insecure"` + // CAFile is the PEM bundle that verifies the collector. Empty uses the + // system roots. + CAFile string `toml:"ca_file" yaml:"ca_file"` + // SampleRatio is the fraction of new traces recorded, 0 to 1. A trace + // started by a caller keeps its caller's decision. Defaulted to 1. + SampleRatio float64 `toml:"sample_ratio" yaml:"sample_ratio"` +} + // ChatConfig configures conversational front-ends. Empty (Telegram.Token.Key == // "") disables chat entirely. type ChatConfig struct { diff --git a/internal/infrastructure/configuration/defaults.go b/internal/infrastructure/configuration/defaults.go index 79f8a8028..9d7a6e1a9 100644 --- a/internal/infrastructure/configuration/defaults.go +++ b/internal/infrastructure/configuration/defaults.go @@ -58,6 +58,9 @@ func (l *Loader) applyDefaults(cfg *config.Config) { if cfg.Health.DependencyTimeout == 0 { cfg.Health.DependencyTimeout = config.Duration(defaultHealthDependencyTimeout) } + if cfg.Telemetry.SampleRatio == 0 { + cfg.Telemetry.SampleRatio = 1 + } l.applyGeneralDefaults(cfg) applyForgeDefaults(cfg) applyDispatchDefaults(cfg) diff --git a/internal/infrastructure/configuration/denied_keys.go b/internal/infrastructure/configuration/denied_keys.go index cff8a7bf3..77228cf59 100644 --- a/internal/infrastructure/configuration/denied_keys.go +++ b/internal/infrastructure/configuration/denied_keys.go @@ -24,6 +24,7 @@ var DeniedKeys = map[string]string{ "services.*": "Binds service listeners and remote clients at startup.", "nats.*": "Constructs the broker connection at startup.", "web.*": "Binds the dashboard listener and authentication at startup.", + "telemetry.*": "Constructs the trace exporter at startup.", "health.*": "Constructs health listeners and probes at startup.", "bindings.encryption_key": "Selects the State Store encryption key at startup.", "bindings.previous_encryption_keys": "Selects the State Store's previous encryption key at startup.", diff --git a/internal/infrastructure/daemonrpc/client.go b/internal/infrastructure/daemonrpc/client.go index 1cb3c4680..9e8f74008 100644 --- a/internal/infrastructure/daemonrpc/client.go +++ b/internal/infrastructure/daemonrpc/client.go @@ -10,6 +10,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) type Client struct{ rpc pb.DaemonServiceClient } @@ -19,7 +20,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) { if err != nil { return nil, nil, err } - conn, err := grpc.NewClient(target, creds) + conn, err := grpc.NewClient(target, creds, telemetry.ClientOption()) if err != nil { return nil, nil, err } diff --git a/internal/infrastructure/gatewayrpc/dial.go b/internal/infrastructure/gatewayrpc/dial.go index 28d17b5c1..17b2cc2ac 100644 --- a/internal/infrastructure/gatewayrpc/dial.go +++ b/internal/infrastructure/gatewayrpc/dial.go @@ -8,6 +8,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) // Dial returns a Gateway chat client for target, authenticated by id. The @@ -23,6 +24,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie } opts := []grpc.DialOption{ creds, + telemetry.ClientOption(), grpc.WithUnaryInterceptor(unaryPrincipal), grpc.WithStreamInterceptor(streamPrincipal), } diff --git a/internal/infrastructure/harnessrpc/harnessrpc.go b/internal/infrastructure/harnessrpc/harnessrpc.go index 6dae5e248..1280fb1e9 100644 --- a/internal/infrastructure/harnessrpc/harnessrpc.go +++ b/internal/infrastructure/harnessrpc/harnessrpc.go @@ -18,6 +18,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) // Client opens sessions against a daemon's HarnessService. @@ -40,7 +41,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie if err != nil { return nil, nil, fmt.Errorf("harness: %w", err) } - conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds}, options...)...) + conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds, telemetry.ClientOption()}, options...)...) if err != nil { return nil, nil, fmt.Errorf("create harness client: %w", err) } diff --git a/internal/infrastructure/messagingrpc/messagingrpc.go b/internal/infrastructure/messagingrpc/messagingrpc.go index 4881f835e..3a4be97ca 100644 --- a/internal/infrastructure/messagingrpc/messagingrpc.go +++ b/internal/infrastructure/messagingrpc/messagingrpc.go @@ -16,6 +16,7 @@ import ( pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1" "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" "github.com/samcharles93/archie-core/internal/logging" ) @@ -79,7 +80,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) { if err != nil { return nil, nil, fmt.Errorf("messaging: %w", err) } - conn, err := grpc.NewClient(target, creds) + conn, err := grpc.NewClient(target, creds, telemetry.ClientOption()) if err != nil { return nil, nil, fmt.Errorf("create messaging client: %w", err) } diff --git a/internal/infrastructure/staterpc/dial.go b/internal/infrastructure/staterpc/dial.go index 6442f220f..daba94d07 100644 --- a/internal/infrastructure/staterpc/dial.go +++ b/internal/infrastructure/staterpc/dial.go @@ -10,6 +10,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) // Dial returns a State Store client for target, attributed to the caller @@ -39,6 +40,7 @@ func DialTask(target string, caPEM []byte, grant string) (*Client, func(), error func dial(target string, creds grpc.DialOption, caller, grant string, options []grpc.DialOption) (*Client, func(), error) { opts := []grpc.DialOption{ creds, + telemetry.ClientOption(), // Keepalive detects a dead peer on an otherwise silent watch stream. grpc.WithKeepaliveParams(ClientKeepaliveParams()), // GetInstalledPackage returns the whole layer, which for an extension is diff --git a/internal/infrastructure/tasktools/client.go b/internal/infrastructure/tasktools/client.go index 9a000554d..019d459b9 100644 --- a/internal/infrastructure/tasktools/client.go +++ b/internal/infrastructure/tasktools/client.go @@ -13,6 +13,7 @@ import ( "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/infrastructure/rpctls" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" + "github.com/samcharles93/archie-core/internal/infrastructure/telemetry" ) // Client never accepts an identity, org or permission list from the harness. @@ -53,7 +54,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) { if err != nil { return nil, nil, err } - conn, err := grpc.NewClient(target, creds, grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20)))) + conn, err := grpc.NewClient(target, creds, telemetry.ClientOption(), grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20)))) if err != nil { return nil, nil, err } diff --git a/internal/infrastructure/telemetry/telemetry.go b/internal/infrastructure/telemetry/telemetry.go new file mode 100644 index 000000000..c16aaf2d7 --- /dev/null +++ b/internal/infrastructure/telemetry/telemetry.go @@ -0,0 +1,137 @@ +// Package telemetry installs the process's OpenTelemetry tracer provider and +// builds the instrumentation options every RPC and HTTP edge shares. +package telemetry + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "os" + "strconv" + "strings" + "time" + + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.43.0" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/samcharles93/archie-core/internal/config" +) + +const shutdownTimeout = 5 * time.Second + +// Start installs the global tracer provider and propagators for component and +// returns the function that flushes and stops them. With no endpoint it +// installs nothing, so the global provider stays a no-op and the returned +// function does nothing. Call it before any server or client is built: the +// instrumentation reads the globals once, at construction. +func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) { + if cfg.Endpoint == "" { + return func() {}, nil + } + res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL, + semconv.ServiceName(component), + semconv.ServiceVersion(version), + semconv.ServiceInstanceID(instanceID()), + )) + if err != nil { + return nil, fmt.Errorf("telemetry resource: %w", err) + } + opts, err := exporterOptions(cfg) + if err != nil { + return nil, err + } + exporter, err := otlptracegrpc.New(ctx, opts...) + if err != nil { + return nil, fmt.Errorf("telemetry exporter: %w", err) + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))), + ) + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + return func() { shutdown(ctx, tp) }, nil +} + +// shutdown flushes after ctx has typically ended, so the deadline hangs off +// a context detached from its cancellation. +func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) { + flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout) + defer cancel() + _ = tp.Shutdown(flush) +} + +func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) { + opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)} + if len(cfg.Headers) > 0 { + opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers)) + } + if cfg.Insecure { + return append(opts, otlptracegrpc.WithInsecure()), nil + } + if cfg.CAFile != "" { + pool, err := caPool(cfg.CAFile) + if err != nil { + return nil, err + } + opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}))) + } + return opts, nil +} + +func caPool(path string) (*x509.CertPool, error) { + pem, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read telemetry ca_file: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("telemetry ca_file holds no certificate") + } + return pool, nil +} + +func instanceID() string { + host, _ := os.Hostname() + return host + "-" + strconv.Itoa(os.Getpid()) +} + +// ServerOption traces every inbound RPC except health probes. +func ServerOption() grpc.ServerOption { + return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck())))) +} + +// ClientOption traces every outbound RPC and sends the span context. +func ClientOption() grpc.DialOption { + return grpc.WithStatsHandler(otelgrpc.NewClientHandler()) +} + +// Handler traces every HTTP request except health probes, naming the span by +// the ServeMux pattern. Place it outermost. +func Handler(next http.Handler) http.Handler { + return otelhttp.NewHandler(next, "http", + otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }), + ) +} + +func isHealth(path string) bool { + return strings.HasPrefix(path, "/health") +} + +// Transport traces outbound HTTP requests made through base. +func Transport(base http.RoundTripper) http.RoundTripper { + return otelhttp.NewTransport(base) +}