diff --git a/CHANGELOG.md b/CHANGELOG.md index 097e806b0..78f45de06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ release's per-component sections are labelled beneath its heading. ### archie-state-store - Six foreign keys now have an index on the referencing column, so deleting a parent row looks its children up instead of scanning the child table: identity aliases and subjects, org memberships and agents, installed package requirements, and the task a handoff came from. +- The control plane no longer polls the database once per watch subscriber: every watcher shares one version read per 250 ms, so an idle instance drops from roughly 180 transactions a second to a handful while a resource change still reaches every watcher on the same schedule. ## [1.62.0] - 2026-11-10 diff --git a/internal/infrastructure/controlplanestore/controlplane.go b/internal/infrastructure/controlplanestore/controlplane.go index 1e199de25..dc4545e14 100644 --- a/internal/infrastructure/controlplanestore/controlplane.go +++ b/internal/infrastructure/controlplanestore/controlplane.go @@ -48,6 +48,7 @@ const defaultHistoryLimit = 200 type Server struct { pb.UnimplementedControlPlaneServiceServer store ResourceStore + watch watchSnapshot definitions map[string]Definition ordered []Definition steps workflow.StepRegistry @@ -163,31 +164,6 @@ func (s *Server) Command(ctx context.Context, request *pb.CommandRequest) (*pb.C return &pb.CommandResponse{Resource: resourceProto(resource)}, nil } -func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error { - if _, ok := s.definitions[request.GetKind()]; !ok { - return status.Error(codes.NotFound, "resource not found") - } - version := request.AfterVersion - ticker := time.NewTicker(250 * time.Millisecond) - defer ticker.Stop() - for { - resource, err := s.resource(stream.Context(), requestOrg(stream.Context()), request.Kind) - if err == nil && resource.Version > version { - if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil { - return err - } - version = resource.Version - } else if err != nil && !errors.Is(err, storecontract.ErrResourceNotFound) { - return mapError(err) - } - select { - case <-stream.Context().Done(): - return stream.Context().Err() - case <-ticker.C: - } - } -} - // orgSeededKinds are the kinds every organisation holds its own document of. // Reading one an org has never written answers with the empty document: an // org that has configured nothing has nothing, rather than the system org's diff --git a/internal/infrastructure/controlplanestore/watch.go b/internal/infrastructure/controlplanestore/watch.go new file mode 100644 index 000000000..5f1a23724 --- /dev/null +++ b/internal/infrastructure/controlplanestore/watch.go @@ -0,0 +1,163 @@ +package controlplanestore + +import ( + "context" + "errors" + "sync" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "github.com/samcharles93/archie-core/internal/contracts/controlplane/v1" + "github.com/samcharles93/archie-core/internal/domain/storecontract" +) + +// watchPollInterval is how often the process refreshes the shared view of +// stored resource versions. Every Watch stream on this Server reads that one +// view, so the State Store's idle database work is one ListResources per window +// however many subscribers exist. Before #1858 each stream ran its own 250 ms +// poll, which was one read per subscriber per tick: about 180 transactions a +// second on an idle instance, near all of them scans of the small resources +// table. The interval is deliberately the old per-stream cadence, so a change +// still reaches a watcher within the same 250 ms it did before. +const watchPollInterval = 250 * time.Millisecond + +// resourceVersionKey identifies one org's copy of a resource kind. +type resourceVersionKey struct { + orgID string + kind string +} + +// watchSnapshot is the control plane's shared view of every stored resource +// version, read from the store at most once per watchPollInterval window. Watch +// streams read it instead of querying the store themselves: the first stream to +// tick in a window refreshes it and the rest of that window's streams reuse the +// result. +// +// An error is shared for the window too, so a store that has stopped answering +// ends each stream once and lets the clients' existing reconnect backoff take +// over, rather than every stream retrying the store every 250 ms. +type watchSnapshot struct { + mu sync.Mutex + window int64 + versions map[resourceVersionKey]int64 + err error +} + +// storedVersions answers every stored resource's version, reading the store at +// most once per watchPollInterval window. The caller reads the returned map +// without a lock: a refresh replaces it and never mutates it. +// +// Streams wait for the window boundary before asking (waitForWatchWindow), so +// the first stream to wake refreshes the snapshot and every other stream reads +// the version it just published. A change committed in one window is therefore +// delivered at the boundary after it, within the interval: the same latency +// bound the per-stream poll had, while the store is asked once per window for +// the whole process. +func (s *Server) storedVersions(ctx context.Context) (map[resourceVersionKey]int64, error) { + snapshot := &s.watch + window := time.Now().UnixNano() / int64(watchPollInterval) + snapshot.mu.Lock() + defer snapshot.mu.Unlock() + if snapshot.window == window { + return snapshot.versions, snapshot.err + } + resources, err := s.store.ListResources(ctx) + snapshot.window, snapshot.versions, snapshot.err = window, nil, nil + if err != nil { + if ctx.Err() != nil { + // A stream that went away while it was the one refreshing the + // snapshot must not hand its cancellation to every other watcher: + // leave the window unfilled so the next stream reads the store again. + snapshot.window = 0 + return nil, err + } + snapshot.err = err + return nil, err + } + versions := make(map[resourceVersionKey]int64, len(resources)) + for _, resource := range resources { + versions[resourceVersionKey{orgID: resource.OrgID, kind: resource.Kind}] = resource.Version + } + snapshot.versions = versions + return versions, nil +} + +// waitForWatchWindow blocks until the next watchPollInterval boundary, +// reporting false when ctx ends first. Every stream in the process wakes just +// after the boundary, so their reads of the shared snapshot land in the same +// window: one store read serves them all, and none of them serves a snapshot +// older than one interval. +func waitForWatchWindow(ctx context.Context) bool { + now := time.Now() + next := now.Truncate(watchPollInterval).Add(watchPollInterval) + timer := time.NewTimer(next.Sub(now)) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +// Watch streams one org's copy of a control-plane resource from the version the +// caller already has. A stream delivers the current document whenever its +// version is newer than the last one delivered, so a reconnect that resumes +// from AfterVersion neither replays a version the client has applied nor skips +// one it has not. Versions written between two polls coalesce into the newest, +// exactly as they did when each stream polled for itself. +func (s *Server) Watch(request *pb.WatchRequest, stream pb.ControlPlaneService_WatchServer) error { + if _, ok := s.definitions[request.GetKind()]; !ok { + return status.Error(codes.NotFound, "resource not found") + } + ctx := stream.Context() + orgID := requestOrg(ctx) + kind := request.GetKind() + version := request.AfterVersion + // The first read is immediate: it seeds an org-seeded kind and answers a + // version written before this stream existed, without waiting a tick. + if err := s.deliverVersion(stream, orgID, kind, &version); err != nil { + return err + } + for { + if !waitForWatchWindow(ctx) { + return ctx.Err() + } + // One snapshot read serves every stream in this process; a stream only + // reads its resource when the snapshot says its kind advanced. + versions, err := s.storedVersions(ctx) + if err != nil { + return mapError(err) + } + if versions[resourceVersionKey{orgID: orgID, kind: kind}] <= version { + continue + } + if err := s.deliverVersion(stream, orgID, kind, &version); err != nil { + return err + } + } +} + +// deliverVersion reads the org's copy of kind and sends it when it is newer +// than *version, the last version this stream delivered, advancing *version on +// a successful send. A kind with no stored value is not an error: the watch +// keeps waiting for the first write. +func (s *Server) deliverVersion(stream pb.ControlPlaneService_WatchServer, orgID, kind string, version *int64) error { + resource, err := s.resource(stream.Context(), orgID, kind) + if errors.Is(err, storecontract.ErrResourceNotFound) { + return nil + } + if err != nil { + return mapError(err) + } + if resource.Version <= *version { + return nil + } + if err := stream.Send(&pb.WatchResponse{Resource: resourceProto(resource)}); err != nil { + return err + } + *version = resource.Version + return nil +}