perf(store): share one control-plane version read across all watchers (#1858)

The control-plane Watch was a 250 ms database poll per subscription: an idle instance ran about 180 transactions a second (archied 14 streams, gateway 5, dashboard 26), almost all sequential scans of the small resources table. The State Store now keeps one shared view of every stored resource version, read once per 250 ms window and reused by every Watch stream; a stream reads its own resource only when its kind's version advanced, so idle database work is one read per interval for the whole process whatever the subscriber count.

The watch contract is unchanged: resume from AfterVersion, no missed update (versions are monotonic per org and kind), the immediate seed probe, the same 250 ms latency bound (streams wake on the same window boundary), and a store error still ends the stream with Unavailable so the clients' existing reconnect backoff applies.
This commit is contained in:
sam committed 2026-11-11 03:03:37 +11:00
1 parent c155bb24ba
commit 161e8f95bf
3 files changed
+165 -25

No files matched your search

+1
View File
@@ -30,6 +30,7 @@ release's per-component sections are labelled beneath its heading.
### archie-state-store
- Six foreign keys now have an index on the referencing column, so deleting a parent row looks its children up instead of scanning the child table: identity aliases and subjects, org memberships and agents, installed package requirements, and the task a handoff came from.
- The control plane no longer polls the database once per watch subscriber: every watcher shares one version read per 250 ms, so an idle instance drops from roughly 180 transactions a second to a handful while a resource change still reaches every watcher on the same schedule.
## [1.62.0] - 2026-11-10
@@ -48,6 +48,7 @@ const defaultHistoryLimit = 200
type Server struct {
pb.UnimplementedControlPlaneServiceServer
store ResourceStore
watch watchSnapshot
definitions map[string]Definition
ordered []Definition
steps workflow.StepRegistry
@@ -163,31 +164,6 @@ func (s *Server) Command(ctx context.Context, request *pb.CommandRequest) (*pb.C
return &pb.CommandResponse{Resource: resourceProto(resource)}, nil
}
func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error {
if _, ok := s.definitions[request.GetKind()]; !ok {
return status.Error(codes.NotFound, "resource not found")
}
version := request.AfterVersion
ticker := time.NewTicker(250 * time.Millisecond)
defer ticker.Stop()
for {
resource, err := s.resource(stream.Context(), requestOrg(stream.Context()), request.Kind)
if err == nil && resource.Version > version {
if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil {
return err
}
version = resource.Version
} else if err != nil && !errors.Is(err, storecontract.ErrResourceNotFound) {
return mapError(err)
}
select {
case <-stream.Context().Done():
return stream.Context().Err()
case <-ticker.C:
}
}
}
// orgSeededKinds are the kinds every organisation holds its own document of.
// Reading one an org has never written answers with the empty document: an
// org that has configured nothing has nothing, rather than the system org's
@@ -0,0 +1,163 @@
package controlplanestore
import (
"context"
"errors"
"sync"
"time"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
)
// watchPollInterval is how often the process refreshes the shared view of
// stored resource versions. Every Watch stream on this Server reads that one
// view, so the State Store's idle database work is one ListResources per window
// however many subscribers exist. Before #1858 each stream ran its own 250 ms
// poll, which was one read per subscriber per tick: about 180 transactions a
// second on an idle instance, near all of them scans of the small resources
// table. The interval is deliberately the old per-stream cadence, so a change
// still reaches a watcher within the same 250 ms it did before.
const watchPollInterval = 250 * time.Millisecond
// resourceVersionKey identifies one org's copy of a resource kind.
type resourceVersionKey struct {
orgID string
kind string
}
// watchSnapshot is the control plane's shared view of every stored resource
// version, read from the store at most once per watchPollInterval window. Watch
// streams read it instead of querying the store themselves: the first stream to
// tick in a window refreshes it and the rest of that window's streams reuse the
// result.
//
// An error is shared for the window too, so a store that has stopped answering
// ends each stream once and lets the clients' existing reconnect backoff take
// over, rather than every stream retrying the store every 250 ms.
type watchSnapshot struct {
mu sync.Mutex
window int64
versions map[resourceVersionKey]int64
err error
}
// storedVersions answers every stored resource's version, reading the store at
// most once per watchPollInterval window. The caller reads the returned map
// without a lock: a refresh replaces it and never mutates it.
//
// Streams wait for the window boundary before asking (waitForWatchWindow), so
// the first stream to wake refreshes the snapshot and every other stream reads
// the version it just published. A change committed in one window is therefore
// delivered at the boundary after it, within the interval: the same latency
// bound the per-stream poll had, while the store is asked once per window for
// the whole process.
func (s *Server) storedVersions(ctx context.Context) (map[resourceVersionKey]int64, error) {
snapshot := &s.watch
window := time.Now().UnixNano() / int64(watchPollInterval)
snapshot.mu.Lock()
defer snapshot.mu.Unlock()
if snapshot.window == window {
return snapshot.versions, snapshot.err
}
resources, err := s.store.ListResources(ctx)
snapshot.window, snapshot.versions, snapshot.err = window, nil, nil
if err != nil {
if ctx.Err() != nil {
// A stream that went away while it was the one refreshing the
// snapshot must not hand its cancellation to every other watcher:
// leave the window unfilled so the next stream reads the store again.
snapshot.window = 0
return nil, err
}
snapshot.err = err
return nil, err
}
versions := make(map[resourceVersionKey]int64, len(resources))
for _, resource := range resources {
versions[resourceVersionKey{orgID: resource.OrgID, kind: resource.Kind}] = resource.Version
}
snapshot.versions = versions
return versions, nil
}
// waitForWatchWindow blocks until the next watchPollInterval boundary,
// reporting false when ctx ends first. Every stream in the process wakes just
// after the boundary, so their reads of the shared snapshot land in the same
// window: one store read serves them all, and none of them serves a snapshot
// older than one interval.
func waitForWatchWindow(ctx context.Context) bool {
now := time.Now()
next := now.Truncate(watchPollInterval).Add(watchPollInterval)
timer := time.NewTimer(next.Sub(now))
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
// Watch streams one org's copy of a control-plane resource from the version the
// caller already has. A stream delivers the current document whenever its
// version is newer than the last one delivered, so a reconnect that resumes
// from AfterVersion neither replays a version the client has applied nor skips
// one it has not. Versions written between two polls coalesce into the newest,
// exactly as they did when each stream polled for itself.
func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error {
if _, ok := s.definitions[request.GetKind()]; !ok {
return status.Error(codes.NotFound, "resource not found")
}
ctx := stream.Context()
orgID := requestOrg(ctx)
kind := request.GetKind()
version := request.AfterVersion
// The first read is immediate: it seeds an org-seeded kind and answers a
// version written before this stream existed, without waiting a tick.
if err := s.deliverVersion(stream, orgID, kind, &version); err != nil {
return err
}
for {
if !waitForWatchWindow(ctx) {
return ctx.Err()
}
// One snapshot read serves every stream in this process; a stream only
// reads its resource when the snapshot says its kind advanced.
versions, err := s.storedVersions(ctx)
if err != nil {
return mapError(err)
}
if versions[resourceVersionKey{orgID: orgID, kind: kind}] <= version {
continue
}
if err := s.deliverVersion(stream, orgID, kind, &version); err != nil {
return err
}
}
}
// deliverVersion reads the org's copy of kind and sends it when it is newer
// than *version, the last version this stream delivered, advancing *version on
// a successful send. A kind with no stored value is not an error: the watch
// keeps waiting for the first write.
func (s *Server) deliverVersion(stream pb.ControlPlaneService_WatchServer, orgID, kind string, version *int64) error {
resource, err := s.resource(stream.Context(), orgID, kind)
if errors.Is(err, storecontract.ErrResourceNotFound) {
return nil
}
if err != nil {
return mapError(err)
}
if resource.Version <= *version {
return nil
}
if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil {
return err
}
*version = resource.Version
return nil
}