perf(store): share one control-plane version read across all watchers (#1858)
The control-plane Watch was a 250 ms database poll per subscription: an idle instance ran about 180 transactions a second (archied 14 streams, gateway 5, dashboard 26), almost all sequential scans of the small resources table. The State Store now keeps one shared view of every stored resource version, read once per 250 ms window and reused by every Watch stream; a stream reads its own resource only when its kind's version advanced, so idle database work is one read per interval for the whole process whatever the subscriber count. The watch contract is unchanged: resume from AfterVersion, no missed update (versions are monotonic per org and kind), the immediate seed probe, the same 250 ms latency bound (streams wake on the same window boundary), and a store error still ends the stream with Unavailable so the clients' existing reconnect backoff applies.
This commit is contained in:
3 files changed
+165
-25
No files matched your search
@@ -30,6 +30,7 @@ release's per-component sections are labelled beneath its heading.
|
||||
### archie-state-store
|
||||
|
||||
- Six foreign keys now have an index on the referencing column, so deleting a parent row looks its children up instead of scanning the child table: identity aliases and subjects, org memberships and agents, installed package requirements, and the task a handoff came from.
|
||||
- The control plane no longer polls the database once per watch subscriber: every watcher shares one version read per 250 ms, so an idle instance drops from roughly 180 transactions a second to a handful while a resource change still reaches every watcher on the same schedule.
|
||||
|
||||
## [1.62.0] - 2026-11-10
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ const defaultHistoryLimit = 200
|
||||
type Server struct {
|
||||
pb.UnimplementedControlPlaneServiceServer
|
||||
store ResourceStore
|
||||
watch watchSnapshot
|
||||
definitions map[string]Definition
|
||||
ordered []Definition
|
||||
steps workflow.StepRegistry
|
||||
@@ -163,31 +164,6 @@ func (s *Server) Command(ctx context.Context, request *pb.CommandRequest) (*pb.C
|
||||
return &pb.CommandResponse{Resource: resourceProto(resource)}, nil
|
||||
}
|
||||
|
||||
func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error {
|
||||
if _, ok := s.definitions[request.GetKind()]; !ok {
|
||||
return status.Error(codes.NotFound, "resource not found")
|
||||
}
|
||||
version := request.AfterVersion
|
||||
ticker := time.NewTicker(250 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
resource, err := s.resource(stream.Context(), requestOrg(stream.Context()), request.Kind)
|
||||
if err == nil && resource.Version > version {
|
||||
if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil {
|
||||
return err
|
||||
}
|
||||
version = resource.Version
|
||||
} else if err != nil && !errors.Is(err, storecontract.ErrResourceNotFound) {
|
||||
return mapError(err)
|
||||
}
|
||||
select {
|
||||
case <-stream.Context().Done():
|
||||
return stream.Context().Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// orgSeededKinds are the kinds every organisation holds its own document of.
|
||||
// Reading one an org has never written answers with the empty document: an
|
||||
// org that has configured nothing has nothing, rather than the system org's
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package controlplanestore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
)
|
||||
|
||||
// watchPollInterval is how often the process refreshes the shared view of
|
||||
// stored resource versions. Every Watch stream on this Server reads that one
|
||||
// view, so the State Store's idle database work is one ListResources per window
|
||||
// however many subscribers exist. Before #1858 each stream ran its own 250 ms
|
||||
// poll, which was one read per subscriber per tick: about 180 transactions a
|
||||
// second on an idle instance, near all of them scans of the small resources
|
||||
// table. The interval is deliberately the old per-stream cadence, so a change
|
||||
// still reaches a watcher within the same 250 ms it did before.
|
||||
const watchPollInterval = 250 * time.Millisecond
|
||||
|
||||
// resourceVersionKey identifies one org's copy of a resource kind.
|
||||
type resourceVersionKey struct {
|
||||
orgID string
|
||||
kind string
|
||||
}
|
||||
|
||||
// watchSnapshot is the control plane's shared view of every stored resource
|
||||
// version, read from the store at most once per watchPollInterval window. Watch
|
||||
// streams read it instead of querying the store themselves: the first stream to
|
||||
// tick in a window refreshes it and the rest of that window's streams reuse the
|
||||
// result.
|
||||
//
|
||||
// An error is shared for the window too, so a store that has stopped answering
|
||||
// ends each stream once and lets the clients' existing reconnect backoff take
|
||||
// over, rather than every stream retrying the store every 250 ms.
|
||||
type watchSnapshot struct {
|
||||
mu sync.Mutex
|
||||
window int64
|
||||
versions map[resourceVersionKey]int64
|
||||
err error
|
||||
}
|
||||
|
||||
// storedVersions answers every stored resource's version, reading the store at
|
||||
// most once per watchPollInterval window. The caller reads the returned map
|
||||
// without a lock: a refresh replaces it and never mutates it.
|
||||
//
|
||||
// Streams wait for the window boundary before asking (waitForWatchWindow), so
|
||||
// the first stream to wake refreshes the snapshot and every other stream reads
|
||||
// the version it just published. A change committed in one window is therefore
|
||||
// delivered at the boundary after it, within the interval: the same latency
|
||||
// bound the per-stream poll had, while the store is asked once per window for
|
||||
// the whole process.
|
||||
func (s *Server) storedVersions(ctx context.Context) (map[resourceVersionKey]int64, error) {
|
||||
snapshot := &s.watch
|
||||
window := time.Now().UnixNano() / int64(watchPollInterval)
|
||||
snapshot.mu.Lock()
|
||||
defer snapshot.mu.Unlock()
|
||||
if snapshot.window == window {
|
||||
return snapshot.versions, snapshot.err
|
||||
}
|
||||
resources, err := s.store.ListResources(ctx)
|
||||
snapshot.window, snapshot.versions, snapshot.err = window, nil, nil
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
// A stream that went away while it was the one refreshing the
|
||||
// snapshot must not hand its cancellation to every other watcher:
|
||||
// leave the window unfilled so the next stream reads the store again.
|
||||
snapshot.window = 0
|
||||
return nil, err
|
||||
}
|
||||
snapshot.err = err
|
||||
return nil, err
|
||||
}
|
||||
versions := make(map[resourceVersionKey]int64, len(resources))
|
||||
for _, resource := range resources {
|
||||
versions[resourceVersionKey{orgID: resource.OrgID, kind: resource.Kind}] = resource.Version
|
||||
}
|
||||
snapshot.versions = versions
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
// waitForWatchWindow blocks until the next watchPollInterval boundary,
|
||||
// reporting false when ctx ends first. Every stream in the process wakes just
|
||||
// after the boundary, so their reads of the shared snapshot land in the same
|
||||
// window: one store read serves them all, and none of them serves a snapshot
|
||||
// older than one interval.
|
||||
func waitForWatchWindow(ctx context.Context) bool {
|
||||
now := time.Now()
|
||||
next := now.Truncate(watchPollInterval).Add(watchPollInterval)
|
||||
timer := time.NewTimer(next.Sub(now))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Watch streams one org's copy of a control-plane resource from the version the
|
||||
// caller already has. A stream delivers the current document whenever its
|
||||
// version is newer than the last one delivered, so a reconnect that resumes
|
||||
// from AfterVersion neither replays a version the client has applied nor skips
|
||||
// one it has not. Versions written between two polls coalesce into the newest,
|
||||
// exactly as they did when each stream polled for itself.
|
||||
func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error {
|
||||
if _, ok := s.definitions[request.GetKind()]; !ok {
|
||||
return status.Error(codes.NotFound, "resource not found")
|
||||
}
|
||||
ctx := stream.Context()
|
||||
orgID := requestOrg(ctx)
|
||||
kind := request.GetKind()
|
||||
version := request.AfterVersion
|
||||
// The first read is immediate: it seeds an org-seeded kind and answers a
|
||||
// version written before this stream existed, without waiting a tick.
|
||||
if err := s.deliverVersion(stream, orgID, kind, &version); err != nil {
|
||||
return err
|
||||
}
|
||||
for {
|
||||
if !waitForWatchWindow(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
// One snapshot read serves every stream in this process; a stream only
|
||||
// reads its resource when the snapshot says its kind advanced.
|
||||
versions, err := s.storedVersions(ctx)
|
||||
if err != nil {
|
||||
return mapError(err)
|
||||
}
|
||||
if versions[resourceVersionKey{orgID: orgID, kind: kind}] <= version {
|
||||
continue
|
||||
}
|
||||
if err := s.deliverVersion(stream, orgID, kind, &version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deliverVersion reads the org's copy of kind and sends it when it is newer
|
||||
// than *version, the last version this stream delivered, advancing *version on
|
||||
// a successful send. A kind with no stored value is not an error: the watch
|
||||
// keeps waiting for the first write.
|
||||
func (s *Server) deliverVersion(stream pb.ControlPlaneService_WatchServer, orgID, kind string, version *int64) error {
|
||||
resource, err := s.resource(stream.Context(), orgID, kind)
|
||||
if errors.Is(err, storecontract.ErrResourceNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return mapError(err)
|
||||
}
|
||||
if resource.Version <= *version {
|
||||
return nil
|
||||
}
|
||||
if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil {
|
||||
return err
|
||||
}
|
||||
*version = resource.Version
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user