feat(servicekit): OpenTelemetry tracer provider and RPC propagation

This commit is contained in:
sam committed 2026-10-25 23:11:03 +11:00
1 parent afd6669877
commit c6187b7c23
23 files changed
+288 -29

No files matched your search

+8
View File
@@ -45,6 +45,14 @@ task_wall_clock = "4h" # a whole task run
listen = "127.0.0.1:8485"
# dependency_timeout = "5s"
# Trace export. Without an endpoint nothing is exported.
# [telemetry]
# endpoint = "127.0.0.1:4317" # OTLP/gRPC collector
# insecure = false # plaintext to the collector
# ca_file = "" # PEM bundle verifying the collector
# sample_ratio = 1.0 # share of new traces recorded
# headers = { authorization = "Bearer ..." }
[web]
listen = "127.0.0.1:8484" # "off" disables the dashboard
# trust_forwarded_headers = false # only behind a proxy you control
+8 -1
View File
@@ -8,6 +8,7 @@ require (
github.com/ProtonMail/go-crypto v1.5.2 // indirect
github.com/antithesishq/antithesis-sdk-go v0.8.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudflare/circl v1.6.5 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
@@ -28,6 +29,7 @@ require (
github.com/go-webauthn/x v0.3.1 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/hashicorp/yamux v0.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -55,9 +57,10 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect
@@ -98,7 +101,11 @@ require (
go.lsp.dev/jsonrpc2 v1.0.1
go.lsp.dev/protocol v1.0.1
go.lsp.dev/uri v1.0.1
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
golang.org/x/crypto v0.58.0
golang.org/x/mod v0.42.0
+14
View File
@@ -20,6 +20,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
github.com/cedar-policy/cedar-go v1.8.0 h1:9gcU7EHXwHC2RMdpph68yTAkdB3behTTssC+kt4GoS8=
github.com/cedar-policy/cedar-go v1.8.0/go.mod h1:h5+3CVW1oI5LXVskJG+my9TFCYI5yjh/+Ul3EJie6MI=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
@@ -96,6 +98,8 @@ github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+
github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs=
@@ -207,10 +211,16 @@ go.lsp.dev/uri v1.0.1 h1:ywxJN1UXoQ+v0I9lkoQNMmsm7lub5G9ZG7KXT8YTx30=
go.lsp.dev/uri v1.0.1/go.mod h1:06a0ghafs4PuqkLsJGUmlgPHw6+kH79edRXjOT/rA8s=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
@@ -221,6 +231,10 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+9 -6
View File
@@ -223,7 +223,7 @@ func (b *boot) cleanup() {
}
}
func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error {
func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
loader, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
if err != nil {
return err
@@ -233,20 +233,23 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error
b.cfg = b.doc.Config
b.cfgHolder = config.NewHolder(b.cfg)
b.currentProvenance.Store(&b.doc.Provenance)
b.setupLogging()
return nil
return b.setupLogging(ctx)
}
// setupLogging stays on stderr for an offline command (stderrLog): creating,
// appending to or rotating the deployment's log is a side effect a diagnosis
// must not have.
func (b *boot) setupLogging() {
func (b *boot) setupLogging(ctx context.Context) error {
if b.stderrLog {
return
return nil
}
logs, err := servicekit.Logging(ctx, b.cfg, "daemon")
if err != nil {
return err
}
logs := servicekit.Logging(b.cfg, "daemon")
b.log, b.logFeed, b.taskLogs = logs.Log, logs.Feed, logs.TaskLogs
b.addCleanup(func() { _ = logs.Closer.Close() })
return nil
}
// openStores wires the secret registry and the conversation store. The
+6 -3
View File
@@ -109,7 +109,7 @@ type server struct {
func Run(ctx context.Context, options Options) error {
b := &server{log: slog.New(slog.NewJSONHandler(os.Stderr, nil))}
defer b.cleanup()
if err := b.loadConfig(options.Config, options.Overlay); err != nil {
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
return err
}
if err := b.openState(ctx); err != nil {
@@ -151,7 +151,7 @@ func (b *server) chatLLM() *runtime.Runtime { return b.llm.Load() }
func (b *server) setLLM(rt *runtime.Runtime) { b.llm.Store(rt) }
func (b *server) announceReady() { sdnotify.Ready(b.log) }
func (b *server) loadConfig(cfgPath, overlayPath string) error {
func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
_, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
if err != nil {
return err
@@ -160,7 +160,10 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
b.cfgPath = cfgPath
b.catalog = modelcatalog.NewCatalog(cfgPath)
b.cfgHolder = config.NewHolder(b.cfg)
logs := servicekit.Logging(b.cfg, "gateway")
logs, err := servicekit.Logging(ctx, b.cfg, "gateway")
if err != nil {
return err
}
b.log, b.logFeed = logs.Log, logs.Feed
b.addCleanup(func() { _ = logs.Closer.Close() })
b.secrets = secret.NewRegistry()
+4
View File
@@ -26,6 +26,7 @@ type ResolvedConfig struct {
WorkDir string
BotUser string
ShowToolCalls bool
Telemetry config.Telemetry
}
type projection struct {
@@ -37,6 +38,7 @@ type projection struct {
workDir string
botUser string
showToolCalls bool
telemetry config.Telemetry
}
// Resolve loads the configuration, extracts the messaging projection and
@@ -71,6 +73,7 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
WorkDir: proj.workDir,
BotUser: proj.botUser,
ShowToolCalls: proj.showToolCalls,
Telemetry: proj.telemetry,
}, nil
}
@@ -95,6 +98,7 @@ func project(cfg config.Config) projection {
workDir: cfg.WorkDir,
botUser: cfg.BotUser,
showToolCalls: cfg.Chat.ShowToolCalls,
telemetry: cfg.Telemetry,
}
}
+17 -6
View File
@@ -10,6 +10,7 @@ import (
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/applystatus"
"github.com/samcharles93/archie-core/internal/domain/messaging"
@@ -22,20 +23,30 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
"github.com/samcharles93/archie-core/internal/logging"
"github.com/samcharles93/archie-core/internal/releaseupdate"
"github.com/samcharles93/archie-core/internal/secret"
)
// Run runs the standalone Messaging Service process until ctx is cancelled.
func Run(ctx context.Context, o Options) error {
log := slog.Default().With("service", "archie-messaging")
log, feed := servicekit.Diagnostics(log, presence.Messaging)
// resolveTraced resolves the configuration and starts trace export from it.
func resolveTraced(ctx context.Context, o Options, log *slog.Logger) (ResolvedConfig, func(), error) {
cfg, err := Resolve(o, log)
if err != nil {
return fmt.Errorf("resolve messaging options: %w", err)
return ResolvedConfig{}, nil, fmt.Errorf("resolve messaging options: %w", err)
}
stop, err := telemetry.Start(ctx, cfg.Telemetry, "messaging", buildinfo.Version)
return cfg, stop, err
}
// Run runs the standalone Messaging Service process until ctx is cancelled.
func Run(ctx context.Context, o Options) error {
log, feed := servicekit.Diagnostics(slog.Default().With("service", "archie-messaging"), presence.Messaging)
cfg, stopTracing, err := resolveTraced(ctx, o, log)
if err != nil {
return err
}
defer stopTracing()
chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Identity)
if err != nil {
+3
View File
@@ -27,6 +27,7 @@ func Resolve(o Options, log *slog.Logger) (Options, error) {
o = merge(o, cfg)
tls = cfg.tls
o.instance = cfg.instance
o.telemetry = cfg.telemetry
}
}
if o.instance == "" {
@@ -70,6 +71,7 @@ type projection struct {
tls config.TLS
capture CaptureOptions
instance string
telemetry config.Telemetry
}
func project(cfg config.Config) projection {
@@ -82,6 +84,7 @@ func project(cfg config.Config) projection {
daemon: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameDaemon).Target},
tls: cfg.TLS,
instance: cfg.Instance,
telemetry: cfg.Telemetry,
capture: CaptureOptions{
Retention: cfg.Capture.Retention.Std(),
MaxEvents: cfg.Capture.MaxEvents,
+3
View File
@@ -5,6 +5,7 @@ package archieui
import (
"time"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
)
@@ -78,6 +79,8 @@ type Options struct {
// instance is the deployment name, from the configuration source or
// ARCHIE_INSTANCE.
instance string
// telemetry is the [telemetry] section, read only by Run.
telemetry config.Telemetry
}
// CaptureOptions carries the [capture] settings the capture receiver needs.
+15 -2
View File
@@ -14,6 +14,7 @@ import (
"time"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/presence"
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
@@ -23,6 +24,7 @@ import (
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/harnessrpc"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
"github.com/samcharles93/archie-core/internal/webui"
)
@@ -34,6 +36,16 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log
return infraaccess.StartLive(ctx, store.ListPolicies, store.PoliciesVersion, log)
}
// resolveTraced resolves the options and starts trace export from them.
func resolveTraced(ctx context.Context, options Options, log *slog.Logger) (Options, func(), error) {
opts, err := Resolve(options, log)
if err != nil {
return Options{}, nil, err
}
stop, err := telemetry.Start(ctx, opts.telemetry, "ui", buildinfo.Version)
return opts, stop, err
}
// Run serves the dashboard until ctx is cancelled. It resolves its own
// options, dials the Gateway and State Store contracts itself, composes the
// HTTP surface over them, and unwinds listener then clients in reverse order
@@ -41,10 +53,11 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log
func Run(ctx context.Context, options Options) error {
log := slog.New(slog.NewJSONHandler(os.Stderr, nil)).With("component", "ui")
log, feed := servicekit.Diagnostics(log, presence.UI)
opts, err := Resolve(options, log)
opts, stopTracing, err := resolveTraced(ctx, options, log)
if err != nil {
return err
}
defer stopTracing()
warnPlainHTTP(log, opts)
var cleanups []func()
defer func() {
@@ -157,7 +170,7 @@ func wireHarness(opts Options, srv *webui.Server, cleanups *[]func()) error {
// serve runs handler until ctx ends, then shuts down within
// ShutdownTimeout.
func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error {
server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout}
server := &http.Server{Handler: telemetry.Handler(handler), ReadHeaderTimeout: opts.ReadHeaderTimeout}
serveErr := make(chan error, 1)
go func() { serveErr <- server.Serve(listener) }()
select {
+22 -5
View File
@@ -1,14 +1,17 @@
package servicekit
import (
"context"
"fmt"
"io"
"log/slog"
"os"
"path/filepath"
"github.com/samcharles93/archie-core/internal/buildinfo"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
"github.com/samcharles93/archie-core/internal/logging"
)
@@ -37,12 +40,18 @@ type Logs struct {
Closer io.Closer
}
// Logging opens the configured log for component. A file log that cannot be
// opened is reported and logging continues on stderr.
func Logging(cfg config.Config, component string) Logs {
// Logging opens the configured log for component and starts trace export. A
// file log that cannot be opened is reported and logging continues on stderr;
// a trace exporter that cannot be built fails startup. Closing flushes the
// exporter, then closes the log, so shutdown records are still written.
func Logging(ctx context.Context, cfg config.Config, component string) (Logs, error) {
stopTracing, err := telemetry.Start(ctx, cfg.Telemetry, component, buildinfo.Version)
if err != nil {
return Logs{}, err
}
feed := logging.NewFeed(1000)
taskLogs := logging.NewTaskRegistry(filepath.Join(cfg.StateDir, "logs", "tasks"), feed, logging.TaskSinkOptions{})
fileLog, closer, err := logging.New(logging.Options{
fileLog, fileCloser, err := logging.New(logging.Options{
File: cfg.Log.File,
MaxSizeMB: cfg.Log.MaxSizeMB,
Keep: cfg.Log.Keep,
@@ -56,9 +65,17 @@ func Logging(cfg config.Config, component string) Logs {
} else if cfg.Log.File != "" {
log.Info("logging to file", "path", cfg.Log.File)
}
return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer}
closer := closerFunc(func() error {
stopTracing()
return fileCloser.Close()
})
return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer}, nil
}
type closerFunc func() error
func (f closerFunc) Close() error { return f() }
// Diagnostics captures a service's stderr log for its recent-log contract.
func Diagnostics(log *slog.Logger, service string) (*slog.Logger, *logging.Feed) {
feed := logging.NewFeed(1000)
+3
View File
@@ -7,6 +7,8 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// maxConcurrentStreams caps the calls one connection holds open at once, so a
@@ -20,6 +22,7 @@ const maxConcurrentStreams = 256
// the OS gives up on the socket.
func ServerLimits() []grpc.ServerOption {
return []grpc.ServerOption{
telemetry.ServerOption(),
grpc.MaxConcurrentStreams(maxConcurrentStreams),
grpc.ConnectionTimeout(10 * time.Second),
grpc.KeepaliveParams(keepalive.ServerParameters{
+5 -2
View File
@@ -48,7 +48,7 @@ func (b *server) cleanup() {
}
}
func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) error {
func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
_, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
if err != nil {
return err
@@ -57,7 +57,10 @@ func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) erro
if b.stderrLog {
return nil
}
logs := servicekit.Logging(b.cfg, "state-store")
logs, err := servicekit.Logging(ctx, b.cfg, "state-store")
if err != nil {
return err
}
b.log, b.taskLogs, b.logFeed = logs.Log, logs.TaskLogs, logs.Feed
b.addCleanup(func() { _ = logs.Closer.Close() })
return nil
+18
View File
@@ -408,6 +408,7 @@ type Config struct {
Web Web `toml:"web" yaml:"web"`
Health Health `toml:"health" yaml:"health"`
Log Log `toml:"log" yaml:"log"`
Telemetry Telemetry `toml:"telemetry" yaml:"telemetry"`
Notify Notify `toml:"notify" yaml:"notify"`
NATS NATSConfig `toml:"nats" yaml:"nats"`
Containers ContainerConfig `toml:"containers" yaml:"containers"`
@@ -884,6 +885,23 @@ type Log struct {
Quiet bool `toml:"quiet" yaml:"quiet"`
}
// Telemetry configures OTLP trace export. An empty Endpoint disables tracing:
// nothing is exported and no tracer provider is installed.
type Telemetry struct {
// Endpoint is the OTLP/gRPC collector address, host:port.
Endpoint string `toml:"endpoint" yaml:"endpoint"`
// Headers are sent with every export, for collector authentication.
Headers map[string]string `toml:"headers" yaml:"headers"`
// Insecure exports over plaintext gRPC instead of TLS.
Insecure bool `toml:"insecure" yaml:"insecure"`
// CAFile is the PEM bundle that verifies the collector. Empty uses the
// system roots.
CAFile string `toml:"ca_file" yaml:"ca_file"`
// SampleRatio is the fraction of new traces recorded, 0 to 1. A trace
// started by a caller keeps its caller's decision. Defaulted to 1.
SampleRatio float64 `toml:"sample_ratio" yaml:"sample_ratio"`
}
// ChatConfig configures conversational front-ends. Empty (Telegram.Token.Key ==
// "") disables chat entirely.
type ChatConfig struct {
@@ -58,6 +58,9 @@ func (l *Loader) applyDefaults(cfg *config.Config) {
if cfg.Health.DependencyTimeout == 0 {
cfg.Health.DependencyTimeout = config.Duration(defaultHealthDependencyTimeout)
}
if cfg.Telemetry.SampleRatio == 0 {
cfg.Telemetry.SampleRatio = 1
}
l.applyGeneralDefaults(cfg)
applyForgeDefaults(cfg)
applyDispatchDefaults(cfg)
@@ -24,6 +24,7 @@ var DeniedKeys = map[string]string{
"services.*": "Binds service listeners and remote clients at startup.",
"nats.*": "Constructs the broker connection at startup.",
"web.*": "Binds the dashboard listener and authentication at startup.",
"telemetry.*": "Constructs the trace exporter at startup.",
"health.*": "Constructs health listeners and probes at startup.",
"bindings.encryption_key": "Selects the State Store encryption key at startup.",
"bindings.previous_encryption_keys": "Selects the State Store's previous encryption key at startup.",
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
type Client struct{ rpc pb.DaemonServiceClient }
@@ -19,7 +20,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
if err != nil {
return nil, nil, err
}
conn, err := grpc.NewClient(target, creds)
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption())
if err != nil {
return nil, nil, err
}
@@ -8,6 +8,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// Dial returns a Gateway chat client for target, authenticated by id. The
@@ -23,6 +24,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie
}
opts := []grpc.DialOption{
creds,
telemetry.ClientOption(),
grpc.WithUnaryInterceptor(unaryPrincipal),
grpc.WithStreamInterceptor(streamPrincipal),
}
@@ -18,6 +18,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// Client opens sessions against a daemon's HarnessService.
@@ -40,7 +41,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie
if err != nil {
return nil, nil, fmt.Errorf("harness: %w", err)
}
conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds}, options...)...)
conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds, telemetry.ClientOption()}, options...)...)
if err != nil {
return nil, nil, fmt.Errorf("create harness client: %w", err)
}
@@ -16,6 +16,7 @@ import (
pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
"github.com/samcharles93/archie-core/internal/logging"
)
@@ -79,7 +80,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
if err != nil {
return nil, nil, fmt.Errorf("messaging: %w", err)
}
conn, err := grpc.NewClient(target, creds)
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption())
if err != nil {
return nil, nil, fmt.Errorf("create messaging client: %w", err)
}
+2
View File
@@ -10,6 +10,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// Dial returns a State Store client for target, attributed to the caller
@@ -39,6 +40,7 @@ func DialTask(target string, caPEM []byte, grant string) (*Client, func(), error
func dial(target string, creds grpc.DialOption, caller, grant string, options []grpc.DialOption) (*Client, func(), error) {
opts := []grpc.DialOption{
creds,
telemetry.ClientOption(),
// Keepalive detects a dead peer on an otherwise silent watch stream.
grpc.WithKeepaliveParams(ClientKeepaliveParams()),
// GetInstalledPackage returns the whole layer, which for an extension is
+2 -1
View File
@@ -13,6 +13,7 @@ import (
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
)
// Client never accepts an identity, org or permission list from the harness.
@@ -53,7 +54,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
if err != nil {
return nil, nil, err
}
conn, err := grpc.NewClient(target, creds, grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20))))
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption(), grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20))))
if err != nil {
return nil, nil, err
}
@@ -0,0 +1,137 @@
// Package telemetry installs the process's OpenTelemetry tracer provider and
// builds the instrumentation options every RPC and HTTP edge shares.
package telemetry
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"github.com/samcharles93/archie-core/internal/config"
)
const shutdownTimeout = 5 * time.Second
// Start installs the global tracer provider and propagators for component and
// returns the function that flushes and stops them. With no endpoint it
// installs nothing, so the global provider stays a no-op and the returned
// function does nothing. Call it before any server or client is built: the
// instrumentation reads the globals once, at construction.
func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) {
if cfg.Endpoint == "" {
return func() {}, nil
}
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
semconv.ServiceName(component),
semconv.ServiceVersion(version),
semconv.ServiceInstanceID(instanceID()),
))
if err != nil {
return nil, fmt.Errorf("telemetry resource: %w", err)
}
opts, err := exporterOptions(cfg)
if err != nil {
return nil, err
}
exporter, err := otlptracegrpc.New(ctx, opts...)
if err != nil {
return nil, fmt.Errorf("telemetry exporter: %w", err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exporter),
sdktrace.WithResource(res),
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))),
)
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
return func() { shutdown(ctx, tp) }, nil
}
// shutdown flushes after ctx has typically ended, so the deadline hangs off
// a context detached from its cancellation.
func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) {
flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout)
defer cancel()
_ = tp.Shutdown(flush)
}
func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) {
opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)}
if len(cfg.Headers) > 0 {
opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers))
}
if cfg.Insecure {
return append(opts, otlptracegrpc.WithInsecure()), nil
}
if cfg.CAFile != "" {
pool, err := caPool(cfg.CAFile)
if err != nil {
return nil, err
}
opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12})))
}
return opts, nil
}
func caPool(path string) (*x509.CertPool, error) {
pem, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read telemetry ca_file: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, errors.New("telemetry ca_file holds no certificate")
}
return pool, nil
}
func instanceID() string {
host, _ := os.Hostname()
return host + "-" + strconv.Itoa(os.Getpid())
}
// ServerOption traces every inbound RPC except health probes.
func ServerOption() grpc.ServerOption {
return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck()))))
}
// ClientOption traces every outbound RPC and sends the span context.
func ClientOption() grpc.DialOption {
return grpc.WithStatsHandler(otelgrpc.NewClientHandler())
}
// Handler traces every HTTP request except health probes, naming the span by
// the ServeMux pattern. Place it outermost.
func Handler(next http.Handler) http.Handler {
return otelhttp.NewHandler(next, "http",
otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }),
)
}
func isHealth(path string) bool {
return strings.HasPrefix(path, "/health")
}
// Transport traces outbound HTTP requests made through base.
func Transport(base http.RoundTripper) http.RoundTripper {
return otelhttp.NewTransport(base)
}