feat(servicekit): OpenTelemetry tracer provider and RPC propagation
This commit is contained in:
1 parent
afd6669877
commit
c6187b7c23
23 files changed
+288
-29
No files matched your search
@@ -45,6 +45,14 @@ task_wall_clock = "4h" # a whole task run
|
||||
listen = "127.0.0.1:8485"
|
||||
# dependency_timeout = "5s"
|
||||
|
||||
# Trace export. Without an endpoint nothing is exported.
|
||||
# [telemetry]
|
||||
# endpoint = "127.0.0.1:4317" # OTLP/gRPC collector
|
||||
# insecure = false # plaintext to the collector
|
||||
# ca_file = "" # PEM bundle verifying the collector
|
||||
# sample_ratio = 1.0 # share of new traces recorded
|
||||
# headers = { authorization = "Bearer ..." }
|
||||
|
||||
[web]
|
||||
listen = "127.0.0.1:8484" # "off" disables the dashboard
|
||||
# trust_forwarded_headers = false # only behind a proxy you control
|
||||
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/ProtonMail/go-crypto v1.5.2 // indirect
|
||||
github.com/antithesishq/antithesis-sdk-go v0.8.0 // indirect
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.6.5 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
@@ -28,6 +29,7 @@ require (
|
||||
github.com/go-webauthn/x v0.3.1 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@@ -55,9 +57,10 @@ require (
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/log v1.47.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.47.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/exp v0.0.0-20261013213113-2ab7f3cbc5fb // indirect
|
||||
@@ -98,7 +101,11 @@ require (
|
||||
go.lsp.dev/jsonrpc2 v1.0.1
|
||||
go.lsp.dev/protocol v1.0.1
|
||||
go.lsp.dev/uri v1.0.1
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.opentelemetry.io/otel v1.47.0
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
|
||||
go.opentelemetry.io/otel/sdk v1.47.0
|
||||
go.opentelemetry.io/otel/trace v1.47.0
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/mod v0.42.0
|
||||
|
||||
@@ -20,6 +20,8 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
|
||||
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
|
||||
github.com/cedar-policy/cedar-go v1.8.0 h1:9gcU7EHXwHC2RMdpph68yTAkdB3behTTssC+kt4GoS8=
|
||||
github.com/cedar-policy/cedar-go v1.8.0/go.mod h1:h5+3CVW1oI5LXVskJG+my9TFCYI5yjh/+Ul3EJie6MI=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
|
||||
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA=
|
||||
@@ -96,6 +98,8 @@ github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+
|
||||
github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
|
||||
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
|
||||
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
|
||||
github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs=
|
||||
@@ -207,10 +211,16 @@ go.lsp.dev/uri v1.0.1 h1:ywxJN1UXoQ+v0I9lkoQNMmsm7lub5G9ZG7KXT8YTx30=
|
||||
go.lsp.dev/uri v1.0.1/go.mod h1:06a0ghafs4PuqkLsJGUmlgPHw6+kH79edRXjOT/rA8s=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0 h1:Tq+E65HaNbZTeqcLw6QUhsV+/EIWdkfUH2CQzcrlE5M=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.72.0/go.mod h1:A+EVhDakAj1waJBPhmTszZaSr08ww8MRRKy1nB1rqck=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
|
||||
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
|
||||
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 h1:julhjPeUH/q/7hinbSdDdqt5h7Zw9YWmRlWRhI0jd54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0/go.mod h1:Ao2mz688LH/tFf0yMAenidq6k2YNSx6SIY2q6jDACck=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0 h1:UFpxOpYPmMNUtOWhdb+nC1WELIgVf8z9higURrbzYU4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0/go.mod h1:YBGjxe3lt0jtXoEXxGrGhv/jM7oUBUDVK7zhAvNyEjc=
|
||||
go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI=
|
||||
go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw=
|
||||
go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY=
|
||||
@@ -221,6 +231,10 @@ go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV
|
||||
go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw=
|
||||
go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM=
|
||||
go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
|
||||
go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
|
||||
@@ -223,7 +223,7 @@ func (b *boot) cleanup() {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error {
|
||||
func (b *boot) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
|
||||
loader, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -233,20 +233,23 @@ func (b *boot) loadConfig(_ context.Context, cfgPath, overlayPath string) error
|
||||
b.cfg = b.doc.Config
|
||||
b.cfgHolder = config.NewHolder(b.cfg)
|
||||
b.currentProvenance.Store(&b.doc.Provenance)
|
||||
b.setupLogging()
|
||||
return nil
|
||||
return b.setupLogging(ctx)
|
||||
}
|
||||
|
||||
// setupLogging stays on stderr for an offline command (stderrLog): creating,
|
||||
// appending to or rotating the deployment's log is a side effect a diagnosis
|
||||
// must not have.
|
||||
func (b *boot) setupLogging() {
|
||||
func (b *boot) setupLogging(ctx context.Context) error {
|
||||
if b.stderrLog {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
logs, err := servicekit.Logging(ctx, b.cfg, "daemon")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logs := servicekit.Logging(b.cfg, "daemon")
|
||||
b.log, b.logFeed, b.taskLogs = logs.Log, logs.Feed, logs.TaskLogs
|
||||
b.addCleanup(func() { _ = logs.Closer.Close() })
|
||||
return nil
|
||||
}
|
||||
|
||||
// openStores wires the secret registry and the conversation store. The
|
||||
|
||||
@@ -109,7 +109,7 @@ type server struct {
|
||||
func Run(ctx context.Context, options Options) error {
|
||||
b := &server{log: slog.New(slog.NewJSONHandler(os.Stderr, nil))}
|
||||
defer b.cleanup()
|
||||
if err := b.loadConfig(options.Config, options.Overlay); err != nil {
|
||||
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.openState(ctx); err != nil {
|
||||
@@ -151,7 +151,7 @@ func (b *server) chatLLM() *runtime.Runtime { return b.llm.Load() }
|
||||
func (b *server) setLLM(rt *runtime.Runtime) { b.llm.Store(rt) }
|
||||
func (b *server) announceReady() { sdnotify.Ready(b.log) }
|
||||
|
||||
func (b *server) loadConfig(cfgPath, overlayPath string) error {
|
||||
func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
|
||||
_, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -160,7 +160,10 @@ func (b *server) loadConfig(cfgPath, overlayPath string) error {
|
||||
b.cfgPath = cfgPath
|
||||
b.catalog = modelcatalog.NewCatalog(cfgPath)
|
||||
b.cfgHolder = config.NewHolder(b.cfg)
|
||||
logs := servicekit.Logging(b.cfg, "gateway")
|
||||
logs, err := servicekit.Logging(ctx, b.cfg, "gateway")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.log, b.logFeed = logs.Log, logs.Feed
|
||||
b.addCleanup(func() { _ = logs.Closer.Close() })
|
||||
b.secrets = secret.NewRegistry()
|
||||
|
||||
@@ -26,6 +26,7 @@ type ResolvedConfig struct {
|
||||
WorkDir string
|
||||
BotUser string
|
||||
ShowToolCalls bool
|
||||
Telemetry config.Telemetry
|
||||
}
|
||||
|
||||
type projection struct {
|
||||
@@ -37,6 +38,7 @@ type projection struct {
|
||||
workDir string
|
||||
botUser string
|
||||
showToolCalls bool
|
||||
telemetry config.Telemetry
|
||||
}
|
||||
|
||||
// Resolve loads the configuration, extracts the messaging projection and
|
||||
@@ -71,6 +73,7 @@ func Resolve(o Options, log *slog.Logger) (ResolvedConfig, error) {
|
||||
WorkDir: proj.workDir,
|
||||
BotUser: proj.botUser,
|
||||
ShowToolCalls: proj.showToolCalls,
|
||||
Telemetry: proj.telemetry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -95,6 +98,7 @@ func project(cfg config.Config) projection {
|
||||
workDir: cfg.WorkDir,
|
||||
botUser: cfg.BotUser,
|
||||
showToolCalls: cfg.Chat.ShowToolCalls,
|
||||
telemetry: cfg.Telemetry,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
@@ -22,20 +23,30 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/secretengine"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
"github.com/samcharles93/archie-core/internal/releaseupdate"
|
||||
"github.com/samcharles93/archie-core/internal/secret"
|
||||
)
|
||||
|
||||
// Run runs the standalone Messaging Service process until ctx is cancelled.
|
||||
func Run(ctx context.Context, o Options) error {
|
||||
log := slog.Default().With("service", "archie-messaging")
|
||||
|
||||
log, feed := servicekit.Diagnostics(log, presence.Messaging)
|
||||
// resolveTraced resolves the configuration and starts trace export from it.
|
||||
func resolveTraced(ctx context.Context, o Options, log *slog.Logger) (ResolvedConfig, func(), error) {
|
||||
cfg, err := Resolve(o, log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve messaging options: %w", err)
|
||||
return ResolvedConfig{}, nil, fmt.Errorf("resolve messaging options: %w", err)
|
||||
}
|
||||
stop, err := telemetry.Start(ctx, cfg.Telemetry, "messaging", buildinfo.Version)
|
||||
return cfg, stop, err
|
||||
}
|
||||
|
||||
// Run runs the standalone Messaging Service process until ctx is cancelled.
|
||||
func Run(ctx context.Context, o Options) error {
|
||||
log, feed := servicekit.Diagnostics(slog.Default().With("service", "archie-messaging"), presence.Messaging)
|
||||
cfg, stopTracing, err := resolveTraced(ctx, o, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopTracing()
|
||||
|
||||
chat, closeGateway, err := gatewayrpc.Dial(cfg.Options.Gateway.Target, cfg.Identity)
|
||||
if err != nil {
|
||||
|
||||
@@ -27,6 +27,7 @@ func Resolve(o Options, log *slog.Logger) (Options, error) {
|
||||
o = merge(o, cfg)
|
||||
tls = cfg.tls
|
||||
o.instance = cfg.instance
|
||||
o.telemetry = cfg.telemetry
|
||||
}
|
||||
}
|
||||
if o.instance == "" {
|
||||
@@ -70,6 +71,7 @@ type projection struct {
|
||||
tls config.TLS
|
||||
capture CaptureOptions
|
||||
instance string
|
||||
telemetry config.Telemetry
|
||||
}
|
||||
|
||||
func project(cfg config.Config) projection {
|
||||
@@ -82,6 +84,7 @@ func project(cfg config.Config) projection {
|
||||
daemon: ServiceTarget{Target: cfg.Services.Get(config.ServiceNameDaemon).Target},
|
||||
tls: cfg.TLS,
|
||||
instance: cfg.Instance,
|
||||
telemetry: cfg.Telemetry,
|
||||
capture: CaptureOptions{
|
||||
Retention: cfg.Capture.Retention.Std(),
|
||||
MaxEvents: cfg.Capture.MaxEvents,
|
||||
|
||||
@@ -5,6 +5,7 @@ package archieui
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
)
|
||||
@@ -78,6 +79,8 @@ type Options struct {
|
||||
// instance is the deployment name, from the configuration source or
|
||||
// ARCHIE_INSTANCE.
|
||||
instance string
|
||||
// telemetry is the [telemetry] section, read only by Run.
|
||||
telemetry config.Telemetry
|
||||
}
|
||||
|
||||
// CaptureOptions carries the [capture] settings the capture receiver needs.
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/harnessrpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
"github.com/samcharles93/archie-core/internal/webui"
|
||||
)
|
||||
|
||||
@@ -34,6 +36,16 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log
|
||||
return infraaccess.StartLive(ctx, store.ListPolicies, store.PoliciesVersion, log)
|
||||
}
|
||||
|
||||
// resolveTraced resolves the options and starts trace export from them.
|
||||
func resolveTraced(ctx context.Context, options Options, log *slog.Logger) (Options, func(), error) {
|
||||
opts, err := Resolve(options, log)
|
||||
if err != nil {
|
||||
return Options{}, nil, err
|
||||
}
|
||||
stop, err := telemetry.Start(ctx, opts.telemetry, "ui", buildinfo.Version)
|
||||
return opts, stop, err
|
||||
}
|
||||
|
||||
// Run serves the dashboard until ctx is cancelled. It resolves its own
|
||||
// options, dials the Gateway and State Store contracts itself, composes the
|
||||
// HTTP surface over them, and unwinds listener then clients in reverse order
|
||||
@@ -41,10 +53,11 @@ func buildAccessChain(ctx context.Context, store *staterpc.Client, log *slog.Log
|
||||
func Run(ctx context.Context, options Options) error {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stderr, nil)).With("component", "ui")
|
||||
log, feed := servicekit.Diagnostics(log, presence.UI)
|
||||
opts, err := Resolve(options, log)
|
||||
opts, stopTracing, err := resolveTraced(ctx, options, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopTracing()
|
||||
warnPlainHTTP(log, opts)
|
||||
var cleanups []func()
|
||||
defer func() {
|
||||
@@ -157,7 +170,7 @@ func wireHarness(opts Options, srv *webui.Server, cleanups *[]func()) error {
|
||||
// serve runs handler until ctx ends, then shuts down within
|
||||
// ShutdownTimeout.
|
||||
func serve(ctx context.Context, listener net.Listener, handler http.Handler, opts Options) error {
|
||||
server := &http.Server{Handler: handler, ReadHeaderTimeout: opts.ReadHeaderTimeout}
|
||||
server := &http.Server{Handler: telemetry.Handler(handler), ReadHeaderTimeout: opts.ReadHeaderTimeout}
|
||||
serveErr := make(chan error, 1)
|
||||
go func() { serveErr <- server.Serve(listener) }()
|
||||
select {
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
package servicekit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
)
|
||||
|
||||
@@ -37,12 +40,18 @@ type Logs struct {
|
||||
Closer io.Closer
|
||||
}
|
||||
|
||||
// Logging opens the configured log for component. A file log that cannot be
|
||||
// opened is reported and logging continues on stderr.
|
||||
func Logging(cfg config.Config, component string) Logs {
|
||||
// Logging opens the configured log for component and starts trace export. A
|
||||
// file log that cannot be opened is reported and logging continues on stderr;
|
||||
// a trace exporter that cannot be built fails startup. Closing flushes the
|
||||
// exporter, then closes the log, so shutdown records are still written.
|
||||
func Logging(ctx context.Context, cfg config.Config, component string) (Logs, error) {
|
||||
stopTracing, err := telemetry.Start(ctx, cfg.Telemetry, component, buildinfo.Version)
|
||||
if err != nil {
|
||||
return Logs{}, err
|
||||
}
|
||||
feed := logging.NewFeed(1000)
|
||||
taskLogs := logging.NewTaskRegistry(filepath.Join(cfg.StateDir, "logs", "tasks"), feed, logging.TaskSinkOptions{})
|
||||
fileLog, closer, err := logging.New(logging.Options{
|
||||
fileLog, fileCloser, err := logging.New(logging.Options{
|
||||
File: cfg.Log.File,
|
||||
MaxSizeMB: cfg.Log.MaxSizeMB,
|
||||
Keep: cfg.Log.Keep,
|
||||
@@ -56,9 +65,17 @@ func Logging(cfg config.Config, component string) Logs {
|
||||
} else if cfg.Log.File != "" {
|
||||
log.Info("logging to file", "path", cfg.Log.File)
|
||||
}
|
||||
return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer}
|
||||
closer := closerFunc(func() error {
|
||||
stopTracing()
|
||||
return fileCloser.Close()
|
||||
})
|
||||
return Logs{Log: log, Feed: feed, TaskLogs: taskLogs, Closer: closer}, nil
|
||||
}
|
||||
|
||||
type closerFunc func() error
|
||||
|
||||
func (f closerFunc) Close() error { return f() }
|
||||
|
||||
// Diagnostics captures a service's stderr log for its recent-log contract.
|
||||
func Diagnostics(log *slog.Logger, service string) (*slog.Logger, *logging.Feed) {
|
||||
feed := logging.NewFeed(1000)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// maxConcurrentStreams caps the calls one connection holds open at once, so a
|
||||
@@ -20,6 +22,7 @@ const maxConcurrentStreams = 256
|
||||
// the OS gives up on the socket.
|
||||
func ServerLimits() []grpc.ServerOption {
|
||||
return []grpc.ServerOption{
|
||||
telemetry.ServerOption(),
|
||||
grpc.MaxConcurrentStreams(maxConcurrentStreams),
|
||||
grpc.ConnectionTimeout(10 * time.Second),
|
||||
grpc.KeepaliveParams(keepalive.ServerParameters{
|
||||
|
||||
@@ -48,7 +48,7 @@ func (b *server) cleanup() {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) error {
|
||||
func (b *server) loadConfig(ctx context.Context, cfgPath, overlayPath string) error {
|
||||
_, doc, err := servicekit.Resolve(b.log, cfgPath, overlayPath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -57,7 +57,10 @@ func (b *server) loadConfig(_ context.Context, cfgPath, overlayPath string) erro
|
||||
if b.stderrLog {
|
||||
return nil
|
||||
}
|
||||
logs := servicekit.Logging(b.cfg, "state-store")
|
||||
logs, err := servicekit.Logging(ctx, b.cfg, "state-store")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.log, b.taskLogs, b.logFeed = logs.Log, logs.TaskLogs, logs.Feed
|
||||
b.addCleanup(func() { _ = logs.Closer.Close() })
|
||||
return nil
|
||||
|
||||
@@ -408,6 +408,7 @@ type Config struct {
|
||||
Web Web `toml:"web" yaml:"web"`
|
||||
Health Health `toml:"health" yaml:"health"`
|
||||
Log Log `toml:"log" yaml:"log"`
|
||||
Telemetry Telemetry `toml:"telemetry" yaml:"telemetry"`
|
||||
Notify Notify `toml:"notify" yaml:"notify"`
|
||||
NATS NATSConfig `toml:"nats" yaml:"nats"`
|
||||
Containers ContainerConfig `toml:"containers" yaml:"containers"`
|
||||
@@ -884,6 +885,23 @@ type Log struct {
|
||||
Quiet bool `toml:"quiet" yaml:"quiet"`
|
||||
}
|
||||
|
||||
// Telemetry configures OTLP trace export. An empty Endpoint disables tracing:
|
||||
// nothing is exported and no tracer provider is installed.
|
||||
type Telemetry struct {
|
||||
// Endpoint is the OTLP/gRPC collector address, host:port.
|
||||
Endpoint string `toml:"endpoint" yaml:"endpoint"`
|
||||
// Headers are sent with every export, for collector authentication.
|
||||
Headers map[string]string `toml:"headers" yaml:"headers"`
|
||||
// Insecure exports over plaintext gRPC instead of TLS.
|
||||
Insecure bool `toml:"insecure" yaml:"insecure"`
|
||||
// CAFile is the PEM bundle that verifies the collector. Empty uses the
|
||||
// system roots.
|
||||
CAFile string `toml:"ca_file" yaml:"ca_file"`
|
||||
// SampleRatio is the fraction of new traces recorded, 0 to 1. A trace
|
||||
// started by a caller keeps its caller's decision. Defaulted to 1.
|
||||
SampleRatio float64 `toml:"sample_ratio" yaml:"sample_ratio"`
|
||||
}
|
||||
|
||||
// ChatConfig configures conversational front-ends. Empty (Telegram.Token.Key ==
|
||||
// "") disables chat entirely.
|
||||
type ChatConfig struct {
|
||||
|
||||
@@ -58,6 +58,9 @@ func (l *Loader) applyDefaults(cfg *config.Config) {
|
||||
if cfg.Health.DependencyTimeout == 0 {
|
||||
cfg.Health.DependencyTimeout = config.Duration(defaultHealthDependencyTimeout)
|
||||
}
|
||||
if cfg.Telemetry.SampleRatio == 0 {
|
||||
cfg.Telemetry.SampleRatio = 1
|
||||
}
|
||||
l.applyGeneralDefaults(cfg)
|
||||
applyForgeDefaults(cfg)
|
||||
applyDispatchDefaults(cfg)
|
||||
|
||||
@@ -24,6 +24,7 @@ var DeniedKeys = map[string]string{
|
||||
"services.*": "Binds service listeners and remote clients at startup.",
|
||||
"nats.*": "Constructs the broker connection at startup.",
|
||||
"web.*": "Binds the dashboard listener and authentication at startup.",
|
||||
"telemetry.*": "Constructs the trace exporter at startup.",
|
||||
"health.*": "Constructs health listeners and probes at startup.",
|
||||
"bindings.encryption_key": "Selects the State Store encryption key at startup.",
|
||||
"bindings.previous_encryption_keys": "Selects the State Store's previous encryption key at startup.",
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
type Client struct{ rpc pb.DaemonServiceClient }
|
||||
@@ -19,7 +20,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
conn, err := grpc.NewClient(target, creds)
|
||||
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// Dial returns a Gateway chat client for target, authenticated by id. The
|
||||
@@ -23,6 +24,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie
|
||||
}
|
||||
opts := []grpc.DialOption{
|
||||
creds,
|
||||
telemetry.ClientOption(),
|
||||
grpc.WithUnaryInterceptor(unaryPrincipal),
|
||||
grpc.WithStreamInterceptor(streamPrincipal),
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// Client opens sessions against a daemon's HarnessService.
|
||||
@@ -40,7 +41,7 @@ func Dial(target string, id *rpctls.Identity, options ...grpc.DialOption) (*Clie
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("harness: %w", err)
|
||||
}
|
||||
conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds}, options...)...)
|
||||
conn, err := grpc.NewClient(target, append([]grpc.DialOption{creds, telemetry.ClientOption()}, options...)...)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create harness client: %w", err)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/messaging/v1"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
"github.com/samcharles93/archie-core/internal/logging"
|
||||
)
|
||||
|
||||
@@ -79,7 +80,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("messaging: %w", err)
|
||||
}
|
||||
conn, err := grpc.NewClient(target, creds)
|
||||
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create messaging client: %w", err)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// Dial returns a State Store client for target, attributed to the caller
|
||||
@@ -39,6 +40,7 @@ func DialTask(target string, caPEM []byte, grant string) (*Client, func(), error
|
||||
func dial(target string, creds grpc.DialOption, caller, grant string, options []grpc.DialOption) (*Client, func(), error) {
|
||||
opts := []grpc.DialOption{
|
||||
creds,
|
||||
telemetry.ClientOption(),
|
||||
// Keepalive detects a dead peer on an otherwise silent watch stream.
|
||||
grpc.WithKeepaliveParams(ClientKeepaliveParams()),
|
||||
// GetInstalledPackage returns the whole layer, which for an extension is
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/rpctls"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/telemetry"
|
||||
)
|
||||
|
||||
// Client never accepts an identity, org or permission list from the harness.
|
||||
@@ -53,7 +54,7 @@ func Dial(target string, id *rpctls.Identity) (*Client, func(), error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
conn, err := grpc.NewClient(target, creds, grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20))))
|
||||
conn, err := grpc.NewClient(target, creds, telemetry.ClientOption(), grpc.WithUnaryInterceptor(staterpc.UnaryClientCredentialInterceptor(presence.Daemon, "")), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(messaging.MaxInboundAttachmentBytes+(1<<20))))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
// Package telemetry installs the process's OpenTelemetry tracer provider and
|
||||
// builds the instrumentation options every RPC and HTTP edge shares.
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
|
||||
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/filters"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.43.0"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
)
|
||||
|
||||
const shutdownTimeout = 5 * time.Second
|
||||
|
||||
// Start installs the global tracer provider and propagators for component and
|
||||
// returns the function that flushes and stops them. With no endpoint it
|
||||
// installs nothing, so the global provider stays a no-op and the returned
|
||||
// function does nothing. Call it before any server or client is built: the
|
||||
// instrumentation reads the globals once, at construction.
|
||||
func Start(ctx context.Context, cfg config.Telemetry, component, version string) (func(), error) {
|
||||
if cfg.Endpoint == "" {
|
||||
return func() {}, nil
|
||||
}
|
||||
res, err := resource.Merge(resource.Default(), resource.NewWithAttributes(semconv.SchemaURL,
|
||||
semconv.ServiceName(component),
|
||||
semconv.ServiceVersion(version),
|
||||
semconv.ServiceInstanceID(instanceID()),
|
||||
))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telemetry resource: %w", err)
|
||||
}
|
||||
opts, err := exporterOptions(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
exporter, err := otlptracegrpc.New(ctx, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("telemetry exporter: %w", err)
|
||||
}
|
||||
tp := sdktrace.NewTracerProvider(
|
||||
sdktrace.WithBatcher(exporter),
|
||||
sdktrace.WithResource(res),
|
||||
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(cfg.SampleRatio))),
|
||||
)
|
||||
otel.SetTracerProvider(tp)
|
||||
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
|
||||
return func() { shutdown(ctx, tp) }, nil
|
||||
}
|
||||
|
||||
// shutdown flushes after ctx has typically ended, so the deadline hangs off
|
||||
// a context detached from its cancellation.
|
||||
func shutdown(ctx context.Context, tp *sdktrace.TracerProvider) {
|
||||
flush, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownTimeout)
|
||||
defer cancel()
|
||||
_ = tp.Shutdown(flush)
|
||||
}
|
||||
|
||||
func exporterOptions(cfg config.Telemetry) ([]otlptracegrpc.Option, error) {
|
||||
opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(cfg.Endpoint)}
|
||||
if len(cfg.Headers) > 0 {
|
||||
opts = append(opts, otlptracegrpc.WithHeaders(cfg.Headers))
|
||||
}
|
||||
if cfg.Insecure {
|
||||
return append(opts, otlptracegrpc.WithInsecure()), nil
|
||||
}
|
||||
if cfg.CAFile != "" {
|
||||
pool, err := caPool(cfg.CAFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12})))
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func caPool(path string) (*x509.CertPool, error) {
|
||||
pem, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read telemetry ca_file: %w", err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, errors.New("telemetry ca_file holds no certificate")
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
func instanceID() string {
|
||||
host, _ := os.Hostname()
|
||||
return host + "-" + strconv.Itoa(os.Getpid())
|
||||
}
|
||||
|
||||
// ServerOption traces every inbound RPC except health probes.
|
||||
func ServerOption() grpc.ServerOption {
|
||||
return grpc.StatsHandler(otelgrpc.NewServerHandler(otelgrpc.WithFilter(filters.Not(filters.HealthCheck()))))
|
||||
}
|
||||
|
||||
// ClientOption traces every outbound RPC and sends the span context.
|
||||
func ClientOption() grpc.DialOption {
|
||||
return grpc.WithStatsHandler(otelgrpc.NewClientHandler())
|
||||
}
|
||||
|
||||
// Handler traces every HTTP request except health probes, naming the span by
|
||||
// the ServeMux pattern. Place it outermost.
|
||||
func Handler(next http.Handler) http.Handler {
|
||||
return otelhttp.NewHandler(next, "http",
|
||||
otelhttp.WithFilter(func(r *http.Request) bool { return !isHealth(r.URL.Path) }),
|
||||
)
|
||||
}
|
||||
|
||||
func isHealth(path string) bool {
|
||||
return strings.HasPrefix(path, "/health")
|
||||
}
|
||||
|
||||
// Transport traces outbound HTTP requests made through base.
|
||||
func Transport(base http.RoundTripper) http.RoundTripper {
|
||||
return otelhttp.NewTransport(base)
|
||||
}
|
||||
Reference in new issue
Block a user