From 40ee9ffafa7d564b640365748d3cdafaae2fd2e9 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Fri, 16 Oct 2026 02:57:23 +1100 Subject: [PATCH] fix(rpc): bound gRPC servers and return listener failures Cap concurrent streams, the handshake and dead peers' streams on every service, drain shutdown for at most 10s, and stop messaging when its listener dies instead of running on without Deliver. --- internal/app/archied/harness.go | 2 +- internal/app/archiegateway/gateway.go | 19 +++-------- internal/app/archiemessaging/run.go | 20 +++++++---- internal/app/servicekit/grpcserver.go | 48 +++++++++++++++++++++++++++ internal/app/statestore/server.go | 18 ++-------- 5 files changed, 70 insertions(+), 37 deletions(-) create mode 100644 internal/app/servicekit/grpcserver.go diff --git a/internal/app/archied/harness.go b/internal/app/archied/harness.go index 277aff9ed..08c585ec3 100644 --- a/internal/app/archied/harness.go +++ b/internal/app/archied/harness.go @@ -213,7 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) { Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) }, TTL: setupSessionTTL, } - srv := grpc.NewServer(creds) + srv := grpc.NewServer(append(servicekit.ServerLimits(), creds)...) pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log}) go func() { if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) { diff --git a/internal/app/archiegateway/gateway.go b/internal/app/archiegateway/gateway.go index 65983d5d4..131103c95 100644 --- a/internal/app/archiegateway/gateway.go +++ b/internal/app/archiegateway/gateway.go @@ -4,11 +4,11 @@ import ( "context" "fmt" "net" - "time" "google.golang.org/grpc" "github.com/samcharles93/archie-core/internal/app/controlplane" + "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/messaging" "github.com/samcharles93/archie-core/internal/events" @@ -113,19 +113,8 @@ func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.Pri } func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error { - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) gatewayrpc.RegisterServer(server, contract, sessions, catalog) - serveErr := make(chan error, 1) - go func() { serveErr <- server.Serve(listener) }() - select { - case <-ctx.Done(): - // A client can keep a streaming RPC open indefinitely. Allow draining, - // then force cancellation before closing the session store and tools. - timer := time.AfterFunc(10*time.Second, server.Stop) - server.GracefulStop() - timer.Stop() - return nil - case err := <-serveErr: - return err - } + // Serve's bounded drain returns before the session store and tools close. + return servicekit.Serve(ctx, server, listener) } diff --git a/internal/app/archiemessaging/run.go b/internal/app/archiemessaging/run.go index 1fece07bf..145ce3c50 100644 --- a/internal/app/archiemessaging/run.go +++ b/internal/app/archiemessaging/run.go @@ -126,16 +126,24 @@ func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service if err != nil { return fmt.Errorf("listen messaging (%s): %w", listen, err) } - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) messagingrpc.RegisterServer(server, srv, feed) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // A dead listener stops the channels too: the process exits and its + // supervisor restarts it, rather than it running on without Deliver. + serveErr := make(chan error, 1) go func() { - if err := server.Serve(listener); err != nil { - log.Error("messaging gRPC server stopped", "err", err) - } + serveErr <- servicekit.Serve(ctx, server, listener) + cancel() }() - defer server.GracefulStop() log.Info("messaging service listening", "listen", listen) - return srv.Start(ctx) + startErr := srv.Start(ctx) + cancel() + if err := <-serveErr; err != nil { + return fmt.Errorf("serve messaging: %w", err) + } + return startErr } // superviseSecretEngines runs the secret-engine extensions and reports their diff --git a/internal/app/servicekit/grpcserver.go b/internal/app/servicekit/grpcserver.go new file mode 100644 index 000000000..c2c9706da --- /dev/null +++ b/internal/app/servicekit/grpcserver.go @@ -0,0 +1,48 @@ +package servicekit + +import ( + "context" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +// maxConcurrentStreams caps the calls one connection holds open at once, so a +// single client cannot pin unbounded server memory. It is far above what a +// service's own clients open: a few watches plus in-flight calls. +const maxConcurrentStreams = 256 + +// ServerLimits bounds what one peer can hold on a service's gRPC server: open +// streams per connection, the handshake, and a dead peer's streams, which the +// server's keepalive pings detect and release instead of holding them until +// the OS gives up on the socket. +func ServerLimits() []grpc.ServerOption { + return []grpc.ServerOption{ + grpc.MaxConcurrentStreams(maxConcurrentStreams), + grpc.ConnectionTimeout(10 * time.Second), + grpc.KeepaliveParams(keepalive.ServerParameters{ + Time: time.Minute, + Timeout: 20 * time.Second, + }), + } +} + +// Serve serves server on listener until ctx ends, then drains in-flight calls +// for up to 10 seconds before cancelling them: a streaming call can stay open +// indefinitely and would otherwise hold shutdown. A listener failure is +// returned, so the process exits rather than staying up with nothing serving. +func Serve(ctx context.Context, server *grpc.Server, listener net.Listener) error { + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(listener) }() + select { + case <-ctx.Done(): + timer := time.AfterFunc(10*time.Second, server.Stop) + server.GracefulStop() + timer.Stop() + return nil + case err := <-serveErr: + return err + } +} diff --git a/internal/app/statestore/server.go b/internal/app/statestore/server.go index 21d66e0b0..4b6f88d14 100644 --- a/internal/app/statestore/server.go +++ b/internal/app/statestore/server.go @@ -11,7 +11,6 @@ import ( "fmt" "net" "net/http" - "time" "github.com/jackc/pgx/v5/pgxpool" "google.golang.org/grpc" @@ -427,22 +426,11 @@ func (b *server) accessDeps(deps *staterpc.Deps) { } // serveStateStore registers the StateStore gRPC service and serves it until -// ctx ends, draining in-flight calls before returning (mirroring serveGateway -// in gateway.go, minus the gateway's session-store argument). +// ctx ends. func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error { - server := grpc.NewServer(opts...) + server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) staterpc.RegisterServer(server, deps) - serveErr := make(chan error, 1) - go func() { serveErr <- server.Serve(listener) }() - select { - case <-ctx.Done(): - timer := time.AfterFunc(10*time.Second, server.Stop) - server.GracefulStop() - timer.Stop() - return nil - case err := <-serveErr: - return err - } + return servicekit.Serve(ctx, server, listener) } // startStateStoreReadiness starts a minimal HTTP readiness surface on