fix(rpc): bound gRPC servers and return listener failures

Cap concurrent streams, the handshake and dead peers' streams on every
service, drain shutdown for at most 10s, and stop messaging when its
listener dies instead of running on without Deliver.
This commit is contained in:
sam committed 2026-10-16 02:57:26 +11:00
1 parent 939b912889
commit 40ee9ffafa
5 files changed
+70 -37

No files matched your search

+1 -1
View File
@@ -213,7 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) {
Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) },
TTL: setupSessionTTL,
}
srv := grpc.NewServer(creds)
srv := grpc.NewServer(append(servicekit.ServerLimits(), creds)...)
pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log})
go func() {
if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+4 -15
View File
@@ -4,11 +4,11 @@ import (
"context"
"fmt"
"net"
"time"
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/controlplane"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/messaging"
"github.com/samcharles93/archie-core/internal/events"
@@ -113,19 +113,8 @@ func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.Pri
}
func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error {
server := grpc.NewServer(opts...)
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
gatewayrpc.RegisterServer(server, contract, sessions, catalog)
serveErr := make(chan error, 1)
go func() { serveErr <- server.Serve(listener) }()
select {
case <-ctx.Done():
// A client can keep a streaming RPC open indefinitely. Allow draining,
// then force cancellation before closing the session store and tools.
timer := time.AfterFunc(10*time.Second, server.Stop)
server.GracefulStop()
timer.Stop()
return nil
case err := <-serveErr:
return err
}
// Serve's bounded drain returns before the session store and tools close.
return servicekit.Serve(ctx, server, listener)
}
+14 -6
View File
@@ -126,16 +126,24 @@ func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service
if err != nil {
return fmt.Errorf("listen messaging (%s): %w", listen, err)
}
server := grpc.NewServer(opts...)
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
messagingrpc.RegisterServer(server, srv, feed)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
// A dead listener stops the channels too: the process exits and its
// supervisor restarts it, rather than it running on without Deliver.
serveErr := make(chan error, 1)
go func() {
if err := server.Serve(listener); err != nil {
log.Error("messaging gRPC server stopped", "err", err)
}
serveErr <- servicekit.Serve(ctx, server, listener)
cancel()
}()
defer server.GracefulStop()
log.Info("messaging service listening", "listen", listen)
return srv.Start(ctx)
startErr := srv.Start(ctx)
cancel()
if err := <-serveErr; err != nil {
return fmt.Errorf("serve messaging: %w", err)
}
return startErr
}
// superviseSecretEngines runs the secret-engine extensions and reports their
+48
View File
@@ -0,0 +1,48 @@
package servicekit
import (
"context"
"net"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"
)
// maxConcurrentStreams caps the calls one connection holds open at once, so a
// single client cannot pin unbounded server memory. It is far above what a
// service's own clients open: a few watches plus in-flight calls.
const maxConcurrentStreams = 256
// ServerLimits bounds what one peer can hold on a service's gRPC server: open
// streams per connection, the handshake, and a dead peer's streams, which the
// server's keepalive pings detect and release instead of holding them until
// the OS gives up on the socket.
func ServerLimits() []grpc.ServerOption {
return []grpc.ServerOption{
grpc.MaxConcurrentStreams(maxConcurrentStreams),
grpc.ConnectionTimeout(10 * time.Second),
grpc.KeepaliveParams(keepalive.ServerParameters{
Time: time.Minute,
Timeout: 20 * time.Second,
}),
}
}
// Serve serves server on listener until ctx ends, then drains in-flight calls
// for up to 10 seconds before cancelling them: a streaming call can stay open
// indefinitely and would otherwise hold shutdown. A listener failure is
// returned, so the process exits rather than staying up with nothing serving.
func Serve(ctx context.Context, server *grpc.Server, listener net.Listener) error {
serveErr := make(chan error, 1)
go func() { serveErr <- server.Serve(listener) }()
select {
case <-ctx.Done():
timer := time.AfterFunc(10*time.Second, server.Stop)
server.GracefulStop()
timer.Stop()
return nil
case err := <-serveErr:
return err
}
}
+3 -15
View File
@@ -11,7 +11,6 @@ import (
"fmt"
"net"
"net/http"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"google.golang.org/grpc"
@@ -427,22 +426,11 @@ func (b *server) accessDeps(deps *staterpc.Deps) {
}
// serveStateStore registers the StateStore gRPC service and serves it until
// ctx ends, draining in-flight calls before returning (mirroring serveGateway
// in gateway.go, minus the gateway's session-store argument).
// ctx ends.
func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error {
server := grpc.NewServer(opts...)
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
staterpc.RegisterServer(server, deps)
serveErr := make(chan error, 1)
go func() { serveErr <- server.Serve(listener) }()
select {
case <-ctx.Done():
timer := time.AfterFunc(10*time.Second, server.Stop)
server.GracefulStop()
timer.Stop()
return nil
case err := <-serveErr:
return err
}
return servicekit.Serve(ctx, server, listener)
}
// startStateStoreReadiness starts a minimal HTTP readiness surface on