diff --git a/internal/app/archied/harness.go b/internal/app/archied/harness.go index 277aff9ed..08c585ec3 100644 --- a/internal/app/archied/harness.go +++ b/internal/app/archied/harness.go @@ -213,7 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) { Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) }, TTL: setupSessionTTL, } - srv := grpc.NewServer(creds) + srv := grpc.NewServer(append(servicekit.ServerLimits(), creds)...) pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log}) go func() { if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) { diff --git a/internal/app/archiegateway/gateway.go b/internal/app/archiegateway/gateway.go index 65983d5d4..131103c95 100644 --- a/internal/app/archiegateway/gateway.go +++ b/internal/app/archiegateway/gateway.go @@ -4,11 +4,11 @@ import ( "context" "fmt" "net" - "time" "google.golang.org/grpc" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/events" @@ -113,19 +113,8 @@ func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.Pri } func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error { - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) gatewayrpc.RegisterServer(server, contract, sessions, catalog) - serveErr := make(chan error, 1) - go func() { serveErr <- server.Serve(listener) }() - select { - case <-ctx.Done(): - // A client can keep a streaming RPC open indefinitely. Allow draining, - // then force cancellation before closing the session store and tools. - timer := time.AfterFunc(10*time.Second, server.Stop) - server.GracefulStop() - timer.Stop() - return nil - case err := <-serveErr: - return err - } + // Serve's bounded drain returns before the session store and tools close. + return servicekit.Serve(ctx, server, listener) } diff --git a/internal/app/archiemessaging/run.go b/internal/app/archiemessaging/run.go index 1fece07bf..145ce3c50 100644 --- a/internal/app/archiemessaging/run.go +++ b/internal/app/archiemessaging/run.go @@ -126,16 +126,24 @@ func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service if err != nil { return fmt.Errorf("listen messaging (%s): %w", listen, err) } - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) messagingrpc.RegisterServer(server, srv, feed) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // A dead listener stops the channels too: the process exits and its + // supervisor restarts it, rather than it running on without Deliver. + serveErr := make(chan error, 1) go func() { - if err := server.Serve(listener); err != nil { - log.Error("messaging gRPC server stopped", "err", err) - } + serveErr <- servicekit.Serve(ctx, server, listener) + cancel() }() - defer server.GracefulStop() log.Info("messaging service listening", "listen", listen) - return srv.Start(ctx) + startErr := srv.Start(ctx) + cancel() + if err := <-serveErr; err != nil { + return fmt.Errorf("serve messaging: %w", err) + } + return startErr } // superviseSecretEngines runs the secret-engine extensions and reports their diff --git a/internal/app/servicekit/grpcserver.go b/internal/app/servicekit/grpcserver.go new file mode 100644 index 000000000..c2c9706da --- /dev/null +++ b/internal/app/servicekit/grpcserver.go @@ -0,0 +1,48 @@ +package servicekit + +import ( + "context" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +// maxConcurrentStreams caps the calls one connection holds open at once, so a +// single client cannot pin unbounded server memory. It is far above what a +// service's own clients open: a few watches plus in-flight calls. +const maxConcurrentStreams = 256 + +// ServerLimits bounds what one peer can hold on a service's gRPC server: open +// streams per connection, the handshake, and a dead peer's streams, which the +// server's keepalive pings detect and release instead of holding them until +// the OS gives up on the socket. +func ServerLimits() []grpc.ServerOption { + return []grpc.ServerOption{ + grpc.MaxConcurrentStreams(maxConcurrentStreams), + grpc.ConnectionTimeout(10 * time.Second), + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: time.Minute, + Timeout: 20 * time.Second, + }), + } +} + +// Serve serves server on listener until ctx ends, then drains in-flight calls +// for up to 10 seconds before cancelling them: a streaming call can stay open +// indefinitely and would otherwise hold shutdown. A listener failure is +// returned, so the process exits rather than staying up with nothing serving. +func Serve(ctx context.Context, server *grpc.Server, listener net.Listener) error { + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(listener) }() + select { + case <-ctx.Done(): + timer := time.AfterFunc(10*time.Second, server.Stop) + server.GracefulStop() + timer.Stop() + return nil + case err := <-serveErr: + return err + } +} diff --git a/internal/app/statestore/server.go b/internal/app/statestore/server.go index 21d66e0b0..4b6f88d14 100644 --- a/internal/app/statestore/server.go +++ b/internal/app/statestore/server.go @@ -11,7 +11,6 @@ import ( "fmt" "net" "net/http" - "time" "github.com/jackc/pgx/v5/pgxpool" "google.golang.org/grpc" @@ -427,22 +426,11 @@ func (b *server) accessDeps(deps *staterpc.Deps) { } // serveStateStore registers the StateStore gRPC service and serves it until -// ctx ends, draining in-flight calls before returning (mirroring serveGateway -// in gateway.go, minus the gateway's session-store argument). +// ctx ends. func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error { - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) staterpc.RegisterServer(server, deps) - serveErr := make(chan error, 1) - go func() { serveErr <- server.Serve(listener) }() - select { - case <-ctx.Done(): - timer := time.AfterFunc(10*time.Second, server.Stop) - server.GracefulStop() - timer.Stop() - return nil - case err := <-serveErr: - return err - } + return servicekit.Serve(ctx, server, listener) } // startStateStoreReadiness starts a minimal HTTP readiness surface on