fix(rpc): bound gRPC servers and return listener failures
Cap concurrent streams, the handshake and dead peers' streams on every service, drain shutdown for at most 10s, and stop messaging when its listener dies instead of running on without Deliver.
This commit is contained in:
1 parent
939b912889
commit
40ee9ffafa
5 files changed
+70
-37
No files matched your search
@@ -213,7 +213,7 @@ func (b *boot) setupHarness(ctx context.Context) {
|
||||
Profile: func(name string) (config.AgentProfile, error) { return b.cfgHolder.Get().Containers.Profile(name) },
|
||||
TTL: setupSessionTTL,
|
||||
}
|
||||
srv := grpc.NewServer(creds)
|
||||
srv := grpc.NewServer(append(servicekit.ServerLimits(), creds)...)
|
||||
pb.RegisterHarnessServiceServer(srv, &harnessServer{sessions: manager, runs: b.runView(), log: b.log})
|
||||
go func() {
|
||||
if err := srv.Serve(ln); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
|
||||
|
||||
@@ -4,11 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/domain/access"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
@@ -113,19 +113,8 @@ func gatewayServerOpts(listen string, id *rpctls.Identity, principals access.Pri
|
||||
}
|
||||
|
||||
func serveGateway(ctx context.Context, listener net.Listener, contract gateway.ChatContract, sessions gateway.SessionStore, catalog gatewayrpc.Catalog, opts []grpc.ServerOption) error {
|
||||
server := grpc.NewServer(opts...)
|
||||
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
|
||||
gatewayrpc.RegisterServer(server, contract, sessions, catalog)
|
||||
serveErr := make(chan error, 1)
|
||||
go func() { serveErr <- server.Serve(listener) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
// A client can keep a streaming RPC open indefinitely. Allow draining,
|
||||
// then force cancellation before closing the session store and tools.
|
||||
timer := time.AfterFunc(10*time.Second, server.Stop)
|
||||
server.GracefulStop()
|
||||
timer.Stop()
|
||||
return nil
|
||||
case err := <-serveErr:
|
||||
return err
|
||||
}
|
||||
// Serve's bounded drain returns before the session store and tools close.
|
||||
return servicekit.Serve(ctx, server, listener)
|
||||
}
|
||||
@@ -126,16 +126,24 @@ func serve(ctx context.Context, listen string, id *rpctls.Identity, srv *Service
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen messaging (%s): %w", listen, err)
|
||||
}
|
||||
server := grpc.NewServer(opts...)
|
||||
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
|
||||
messagingrpc.RegisterServer(server, srv, feed)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
// A dead listener stops the channels too: the process exits and its
|
||||
// supervisor restarts it, rather than it running on without Deliver.
|
||||
serveErr := make(chan error, 1)
|
||||
go func() {
|
||||
if err := server.Serve(listener); err != nil {
|
||||
log.Error("messaging gRPC server stopped", "err", err)
|
||||
}
|
||||
serveErr <- servicekit.Serve(ctx, server, listener)
|
||||
cancel()
|
||||
}()
|
||||
defer server.GracefulStop()
|
||||
log.Info("messaging service listening", "listen", listen)
|
||||
return srv.Start(ctx)
|
||||
startErr := srv.Start(ctx)
|
||||
cancel()
|
||||
if err := <-serveErr; err != nil {
|
||||
return fmt.Errorf("serve messaging: %w", err)
|
||||
}
|
||||
return startErr
|
||||
}
|
||||
|
||||
// superviseSecretEngines runs the secret-engine extensions and reports their
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package servicekit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/keepalive"
|
||||
)
|
||||
|
||||
// maxConcurrentStreams caps the calls one connection holds open at once, so a
|
||||
// single client cannot pin unbounded server memory. It is far above what a
|
||||
// service's own clients open: a few watches plus in-flight calls.
|
||||
const maxConcurrentStreams = 256
|
||||
|
||||
// ServerLimits bounds what one peer can hold on a service's gRPC server: open
|
||||
// streams per connection, the handshake, and a dead peer's streams, which the
|
||||
// server's keepalive pings detect and release instead of holding them until
|
||||
// the OS gives up on the socket.
|
||||
func ServerLimits() []grpc.ServerOption {
|
||||
return []grpc.ServerOption{
|
||||
grpc.MaxConcurrentStreams(maxConcurrentStreams),
|
||||
grpc.ConnectionTimeout(10 * time.Second),
|
||||
grpc.KeepaliveParams(keepalive.ServerParameters{
|
||||
Time: time.Minute,
|
||||
Timeout: 20 * time.Second,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// Serve serves server on listener until ctx ends, then drains in-flight calls
|
||||
// for up to 10 seconds before cancelling them: a streaming call can stay open
|
||||
// indefinitely and would otherwise hold shutdown. A listener failure is
|
||||
// returned, so the process exits rather than staying up with nothing serving.
|
||||
func Serve(ctx context.Context, server *grpc.Server, listener net.Listener) error {
|
||||
serveErr := make(chan error, 1)
|
||||
go func() { serveErr <- server.Serve(listener) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer := time.AfterFunc(10*time.Second, server.Stop)
|
||||
server.GracefulStop()
|
||||
timer.Stop()
|
||||
return nil
|
||||
case err := <-serveErr:
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"google.golang.org/grpc"
|
||||
@@ -427,22 +426,11 @@ func (b *server) accessDeps(deps *staterpc.Deps) {
|
||||
}
|
||||
|
||||
// serveStateStore registers the StateStore gRPC service and serves it until
|
||||
// ctx ends, draining in-flight calls before returning (mirroring serveGateway
|
||||
// in gateway.go, minus the gateway's session-store argument).
|
||||
// ctx ends.
|
||||
func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error {
|
||||
server := grpc.NewServer(opts...)
|
||||
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
|
||||
staterpc.RegisterServer(server, deps)
|
||||
serveErr := make(chan error, 1)
|
||||
go func() { serveErr <- server.Serve(listener) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer := time.AfterFunc(10*time.Second, server.Stop)
|
||||
server.GracefulStop()
|
||||
timer.Stop()
|
||||
return nil
|
||||
case err := <-serveErr:
|
||||
return err
|
||||
}
|
||||
return servicekit.Serve(ctx, server, listener)
|
||||
}
|
||||
|
||||
// startStateStoreReadiness starts a minimal HTTP readiness surface on
|
||||
|
||||
Reference in new issue
Block a user