// state_store.go composes the standalone archie-state-store process. It mirrors // gateway.go / cmd/archie-gateway: the binary is a thin main that passes // process inputs into Run, which serves the task and event-capture // stores from Postgres, registers every StateStore gRPC handler (the same service .4.2 serves // in-process on the daemon), and shuts down cleanly on ctx cancellation. package statestore import ( "context" "encoding/json" "fmt" "net" "net/http" "github.com/jackc/pgx/v5/pgxpool" "google.golang.org/grpc" "github.com/samcharles93/archie-core/internal/app/controlplane" "github.com/samcharles93/archie-core/internal/app/servicekit" "github.com/samcharles93/archie-core/internal/config" "github.com/samcharles93/archie-core/internal/domain/access" "github.com/samcharles93/archie-core/internal/domain/alert" "github.com/samcharles93/archie-core/internal/domain/health" "github.com/samcharles93/archie-core/internal/domain/identity" "github.com/samcharles93/archie-core/internal/domain/org" "github.com/samcharles93/archie-core/internal/domain/presence" "github.com/samcharles93/archie-core/internal/domain/storecontract" "github.com/samcharles93/archie-core/internal/domain/storepkg" "github.com/samcharles93/archie-core/internal/domain/usage" infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access" "github.com/samcharles93/archie-core/internal/infrastructure/postgres" "github.com/samcharles93/archie-core/internal/infrastructure/readiness" "github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity" "github.com/samcharles93/archie-core/internal/infrastructure/staterpc" registry "github.com/samcharles93/archie-core/internal/infrastructure/storepkg" "github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps" "github.com/samcharles93/archie-core/internal/secret" ) // Options contains process inputs for the standalone State Store. type Options struct { Config string Overlay string // Listen is the gRPC listen address. Without [tls] it must be loopback. Listen string // ReadyAddr, when non-empty, starts an HTTP readiness surface on the // address: GET /healthz (liveness) and GET /health/detailed (the // state_db probe). Empty disables it. ReadyAddr string } // Run serves the task and event-capture stores from Postgres, // registers every StateStore gRPC handler, and serves until ctx is cancelled. // The conversation store belongs to the separate archie-gateway process. The // store service owns its own DB lifecycle, so // b.cleanup() is the sole owner closing b.st here (in-process owner). func Run(ctx context.Context, options Options) error { //nolint:cyclop // the composition root's setup sequence is deliberately flat and sequential b := newServer() defer b.cleanup() if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil { return err } listen, err := servicekit.ResolveListen("state", options.Listen, b.cfg.Services.Get(config.ServiceNameState).Listen) if err != nil { return err } b.log = b.log.With("component", "state-store", "listen", listen) if err := b.openStateStore(ctx); err != nil { return err } identityStore, ok := b.st.(interface { BootstrapIdentities(context.Context, []string) error }) if !ok { return fmt.Errorf("state store does not support identity bootstrap") } legacyNames := servicekit.IdentityNames(b.cfg) if err := identityStore.BootstrapIdentities(ctx, legacyNames); err != nil { return fmt.Errorf("bootstrap identities: %w", err) } // The resumable default org/workspace upgrade runs before this process // serves anything: the State Store refuses scoped calls until its upgrade // has finished, rather than serving records with no org. // A store // without the upgrader is one that owns no tenant boundary yet, so the // absence degrades to "no upgrade" rather than aborting boot. if upgrader, ok := b.st.(org.Upgrader); ok { if err := upgrader.UpgradeDefaultOrg(ctx); err != nil { return fmt.Errorf("upgrade default org: %w", err) } } if err := b.seedAndValidatePolicies(ctx); err != nil { return err } // The validating side's control plane, built here at the composition root // before the first definition is read or replaced. resources, ok := b.st.(controlplane.ResourceStore) if !ok { return fmt.Errorf("state store does not support control-plane resources") } control, err := openStateStoreControlPlane(resources) if err != nil { return err } _, skipped, err := control.ImportConfig(ctx, b.cfg) if err != nil { return fmt.Errorf("import control-plane resources: %w", err) } b.reportUnseededResources(skipped) grants, err := b.runCredentials(ctx) if err != nil { return err } //nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; TaskGrants.StreamInterceptor derives its context from stream.Context() instead opts, err := stateStoreServerOpts(listen, b.cfg.TLS, grants, b.callers()) if err != nil { return err } listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen) if err != nil { return fmt.Errorf("listen for state store: %w", err) } defer listener.Close() b.log.Info("archie-state-store running", "addr", listener.Addr().String(), "tls", b.cfg.TLS.CertDir != "") if err := b.startOptionalSurfaces(ctx, options); err != nil { return err } // Boot is over and every listener this process serves is bound: the gRPC // contract below and, when -ready-addr asked for one, the readiness HTTP // surface. systemd's READY=1 asserts the same fact, so the announcement // goes out here rather than from a second notion of "started". // Not earlier: a readiness surface that fails to bind // fails the boot, and READY must not precede it. Not later: Serve begins // accepting the moment it is called. b.announceReady() deps := b.stateStoreDeps(grants) controlPlaneDeps(&deps, control) b.startSweeps(ctx, deps) return serveStateStore(ctx, listener, deps, opts) } // seedAndValidatePolicies seeds the shipped role policies for every org that // lacks them, grants the agent identities their org's shipped developer role, // and re-validates every stored policy this process serves. An invalid // policy at any level keeps denying its level and is named as a problem // by the engines the consumers build; the alert sweep raises an alert for it, // so boot goes on and the dashboard can repair it. A store without the access // surfaces degrades: there is no chain to seed. func (b *server) seedAndValidatePolicies(ctx context.Context) error { policies, ok := b.st.(access.PolicyStore) if !ok { return nil } if agents, ok := b.st.(interface { EnsureAgentOrgMembership(context.Context) error }); ok { if err := agents.EnsureAgentOrgMembership(ctx); err != nil { return fmt.Errorf("grant agent org membership: %w", err) } } if orgs, ok := b.st.(org.Repository); ok { list, err := orgs.ListOrgs(ctx) if err != nil { return fmt.Errorf("list orgs for policy seeding: %w", err) } for _, o := range list { if err := policies.EnsureShippedOrgPolicies(ctx, o.ID); err != nil { return fmt.Errorf("seed shipped org policies for %s: %w", o.ID, err) } } } stored, err := policies.ListPolicies(ctx) if err != nil { return fmt.Errorf("load stored policies: %w", err) } engine := infraaccess.New(stored) for _, problem := range engine.Problems() { b.log.Error("stored access policy is invalid and denies its level", "policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err) } return nil } // reportUnseededResources logs every kind ImportConfig could not seed. // // Reported, not fatal: a stale TOML value in a setting the database owns must // not stop this process from starting. The kind stays ABSENT rather than // half-written, so the file document's value is the one in effect for every // reader -- the layering leaves a kind with no stored value alone // (controlplane.resourceReader) -- and the operator's fix is still the file. // Whether the value may RUN is decided where it is used: boot.runtimeConfig // validates the effective document and refuses to start archied or the Gateway // with one it cannot run. The seed is retried on this process's next start, so // a corrected config.toml takes effect when archie-state-store restarts. // // It is a method rather than a loop in Run because the boot sequence // is at its complexity budget: the branch belongs to the report's own policy, // not to the sequence that triggers it. func (b *server) reportUnseededResources(skipped []controlplane.SeedSkip) { for _, skip := range skipped { b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect", "kind", skip.Kind, "err", skip.Err) } } // openStateStoreControlPlane builds the State Store's control plane server: the // validating side of the workflow step vocabulary, registered here at the // composition root before the first definition is read or replaced. The same // provider set (internal/infrastructure/workflowsteps) is what archie-agent // registers before it compiles one, so within one build a definition this // server admits is compilable there; a skewed deploy is only fixed by a // matching deploy. func openStateStoreControlPlane(resources controlplane.ResourceStore) (*controlplane.Server, error) { steps, err := workflowsteps.NewManager() if err != nil { return nil, fmt.Errorf("register workflow step vocabulary: %w", err) } server, err := controlplane.NewServer(resources, steps) if err != nil { return nil, fmt.Errorf("build control plane server: %w", err) } return server, nil } // openStateStore composes the State Store process's persistence: it resolves // secrets (for the binding cipher), opens and migrates the process-scoped // PostgreSQL pool (fail closed), and serves the task and event-capture stores // from it. // It does NOT open gateway chat sessions -- those live in the separate // archie-gateway process on their own store and are out of state-store scope. func (b *server) openStateStore(ctx context.Context) error { cfg, log := b.cfg, b.log // The State Store serves the installed packages extension engines come from, // so it resolves its own references through the env engine only. secrets := secret.NewRegistry() b.secrets = secrets bindingCipher, err := bindingCipherFromConfig(cfg, secrets) if err != nil { log.Error("configure bindings cipher", "err", err) return err } // Open the PostgreSQL pool and apply the schema before anything else: the // State Store fails closed without a working database_url. if err := b.openStateStorePool(ctx); err != nil { return err } b.openTaskStore() b.openEDAStore(bindingCipher) return nil } // stateStoreDeps assembles the store surfaces the StateStore service fronts. // b.st is the narrow storecontract.TaskStore; the concrete store also implements // the capture/mapping/binding surfaces, so each is asserted here (the same // pattern the daemon's wireWebStoreSurfaces uses) and a store that lacks one // degrades that group rather than aborting boot. func (b *server) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps { deps := staterpc.Deps{ LogFeed: b.logFeed, Tasks: b.st, Log: b.log, Grants: grants, // The same root archied dispatches as (servicekit.IdentityNames). RootIdentity: identity.StableID(servicekit.IdentityNames(b.cfg)[0]), } if b.pg != nil { packages := storepkg.Service{ Registry: registry.OCIRegistry{}, Store: postgres.NewInstalledPackages(b.pg), Catalogue: registry.DefaultCatalogue(), } // A projection failure degrades the org resources a package would have // contributed to, the way a group above degrades: the State Store boots // and reports, rather than refusing to serve the task tables. projections, err := newPackageProjections(b.pg) if err != nil { b.log.Error("build package projections", "err", err) } else { packages.Projections = projections } deps.Packages = packages } if identities, ok := b.st.(identity.Repository); ok { deps.Identities = identities } // Subject bindings are asserted separately from the repository facade: only // the authenticating path resolves a verified subject, and a store that // cannot is one where the dashboard cannot name its caller rather than one // that fails to boot. if bindings, ok := b.st.(identity.SubjectBinding); ok { deps.SubjectBindings = bindings } if tokens, ok := b.st.(identity.PersonalTokenStore); ok { deps.PersonalTokens = tokens } // Task logs live in the state directory, which this process owns, and the // dashboard process owns no such directory -- so this is where a task-log // read is served from. The reader is // the daemon's own registry over the configured state_dir. // // The nil check is on the registry, not on the interface it is assigned // to: a nil *logging.TaskRegistry stored in this field produces a non-nil // interface holding a nil pointer, which the server's own // `deps.TaskLogs == nil` guard would not catch. Leaving the field unset is // what makes the RPC answer Unavailable -- the honest "this process cannot // read logs" -- instead of depending on nil-receiver behaviour to notice. if b.taskLogs != nil { deps.TaskLogs = orgTaskLogs{TaskLogStore: b.taskLogs, tasks: b.st} } // The event-capture contracts are served by the event-capture store, not // the task store. BindingTaskCreator stays below on the // task store, because creating a task is the one thing it still does. if b.eda != nil { deps.Captures = b.eda deps.Refusals = b.eda deps.Mappings = b.eda deps.Bindings = b.eda deps.BindingDispatcher = b.eda deps.PlaybookDispatcher = b.eda deps.EventTypes = b.eda deps.Sources = b.eda deps.MappingMatches = b.eda // HarnessSecrets rides the same encrypted-at-rest store as source // webhook secrets, under its // own domain separator (bindingcipher.HarnessSecretDomain). deps.HarnessSecrets = b.eda } // tool_call events project into the tool_calls collection on the same // event-capture store: this process legitimately owns both, so the // projection rides on the task-store surface it decorates. With no // event-capture store (or no task store) there is nothing to project // through, and Tasks passes through unwrapped rather than decorating a // nil operand. if b.eda != nil && b.st != nil { deps.Tasks = newToolCallProjectingTaskStore(b.st, b.eda, b.log) } if btc, ok := b.st.(storecontract.BindingTaskCreator); ok { deps.BindingTaskCreator = btc } b.executionDeps(&deps) b.accessDeps(&deps) if css, ok := b.st.(storecontract.ConfigSnapshotStore); ok { deps.ConfigSnapshots = css } if ps, ok := b.st.(storecontract.PresenceStore); ok { deps.Presence = ps } if as, ok := b.st.(storecontract.ApplyStatusStore); ok { deps.ApplyStatus = as } if cs, ok := b.st.(storecontract.ChannelStatusStore); ok { deps.ChannelStatus = cs } return deps } // executionDeps wires the execution-tree surfaces (workflow calls, step // recording/reading, cancellation) from the same store, degrading each one // independently the store lacks it rather than failing the boot. func (b *server) executionDeps(deps *staterpc.Deps) { if wc, ok := b.st.(storecontract.WorkflowCaller); ok { deps.WorkflowCalls = wc } if sr, ok := b.st.(storecontract.StepRecorder); ok { deps.Steps = sr } if ec, ok := b.st.(storecontract.ExecutionCanceller); ok { deps.Canceller = ec } if ur, ok := b.st.(usage.Recorder); ok { deps.Usage = ur } if sr, ok := b.st.(storecontract.StepReader); ok { deps.StepReader = sr } } // runCredentials builds the run-credential check over this store and drops // the credentials that expired while nothing revoked them. func (b *server) runCredentials(ctx context.Context) (*staterpc.TaskGrants, error) { store, ok := b.st.(storecontract.RunCredentialStore) if !ok { return nil, fmt.Errorf("state store does not support run credentials") } grants := &staterpc.TaskGrants{Store: store} if err := grants.Sweep(ctx); err != nil { return nil, fmt.Errorf("sweep expired run credentials: %w", err) } return grants, nil } // callers resolves the service and principal of each call against this // store's principals. func (b *server) callers() rpcidentity.Callers { ps, _ := b.st.(access.PrincipalSource) return rpcidentity.Callers{Principals: ps} } // accessDeps wires the policy chain and its denial records from the same // store. A store without them -- one that owns // no tenant boundary yet -- degrades the access RPCs rather than failing the // boot, the same pattern the other optional surfaces use. func (b *server) accessDeps(deps *staterpc.Deps) { if accounts, ok := b.st.(identity.Accounts); ok { deps.Accounts = accounts } if ps, ok := b.st.(access.PrincipalSource); ok { deps.Principals = ps } if ps, ok := b.st.(access.PolicyStore); ok { deps.Policies = ps if owners, ok := b.st.(ownerDirectory); ok { deps.Policies = guardedPolicies{PolicyStore: ps, owners: owners} } } if ds, ok := b.st.(access.DenialStore); ok { deps.Denials = ds } if os, ok := b.st.(org.Repository); ok { deps.Orgs = os } if as, ok := b.st.(alert.Store); ok { deps.Alerts = as } } // serveStateStore registers the StateStore gRPC service and serves it until // ctx ends. func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error { server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...) staterpc.RegisterServer(server, deps) return servicekit.Serve(ctx, server, listener) } // startStateStoreReadiness starts a minimal HTTP readiness surface on // readyAddr: GET /healthz and /health answer 200 as a liveness probe (the // process is up), while GET /health/detailed runs the state_db readiness probe // and answers 503 when the store is degraded. It is the standalone process's // counterpart to the daemon's /healthz + /health/detailed (internal/webui). func (b *server) startStateStoreReadiness(ctx context.Context, readyAddr string) error { registry := b.readinessRegistry() mux := http.NewServeMux() mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }) mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) }) mux.HandleFunc("GET /health/detailed", func(w http.ResponseWriter, r *http.Request) { report := registry.Run(r.Context()) w.Header().Set("Content-Type", "application/json") if report.Status != health.StatusOK { w.WriteHeader(http.StatusServiceUnavailable) } _ = json.NewEncoder(w).Encode(report) }) return b.serveHealth(ctx, readyAddr, mux, "state store readiness") } func (b *server) readinessRegistry() *health.Registry { return health.NewRegistry(readiness.NewStoreProbe(b.st)) } // startOptionalSurfaces starts the HTTP surfaces this process serves beside // the gRPC contract, each enabled only by its own flag. // // They are gathered here rather than branched inline because Run is // at its complexity budget: a surface that is off by default belongs to its // own decision, not to the boot sequence that triggers it -- the same reason // reportUnseededResources is a method. func (b *server) startOptionalSurfaces(ctx context.Context, options Options) error { if ps, ok := b.st.(storecontract.PresenceStore); ok { go presence.Run(ctx, presence.StateStore, servicekit.Build(), ps, b.readinessRegistry(), b.log) } if options.ReadyAddr != "" { if err := b.startStateStoreReadiness(ctx, options.ReadyAddr); err != nil { return err } } return nil } // stateStoreServerOpts serves listen with this process's [tls] certificate. A client // certificate is optional here alone: a task container has none and is // confined by its grant. Authentication runs before attribution. // // The keepalive enforcement policy matches staterpc.Dial's clients, whose // idle watches grpc's default policy would answer with GOAWAY // too_many_pings. See staterpc.ServerKeepaliveOption. func stateStoreServerOpts(listen string, tls config.TLS, grants *staterpc.TaskGrants, callers rpcidentity.Callers) ([]grpc.ServerOption, error) { id, err := servicekit.Identity(tls, presence.StateStore) if err != nil { return nil, err } creds, err := id.ServerOption(listen, true) if err != nil { return nil, fmt.Errorf("state store: %w", err) } return []grpc.ServerOption{ creds, staterpc.ServerKeepaliveOption(), grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(), callers.Unary()), grpc.ChainStreamInterceptor(grants.StreamInterceptor(), callers.Stream()), }, nil } // newPackageProjections builds the family projectors an installed package's // contributions apply through: workflows, playbooks, skills and agent profiles become // the org resources the runtime reads live. The step vocabulary // (stepVocabulary) is the same one this process's control plane validates // dashboard edits with, so a definition the projector admits is compilable // where it dispatches (openStateStoreControlPlane for the pairing). func newPackageProjections(pool *pgxpool.Pool) (map[string]storepkg.FamilyProjector, error) { steps, err := workflowsteps.NewManager() if err != nil { return nil, fmt.Errorf("register workflow step vocabulary: %w", err) } workflows, err := controlplane.NewWorkflowPackageProjector( postgres.NewResources(pool), postgres.NewPackageContributions(pool), steps) if err != nil { return nil, fmt.Errorf("build workflow package projector: %w", err) } resources, ledger := postgres.NewResources(pool), postgres.NewPackageContributions(pool) playbooks, err := controlplane.NewPlaybookPackageProjector(resources, ledger) if err != nil { return nil, fmt.Errorf("build playbook package projector: %w", err) } profiles, err := controlplane.NewProfilePackageProjector(resources, ledger) if err != nil { return nil, fmt.Errorf("build profile package projector: %w", err) } skills, err := controlplane.NewSkillPackageProjector(resources, ledger) if err != nil { return nil, fmt.Errorf("build skill package projector: %w", err) } return map[string]storepkg.FamilyProjector{ storepkg.FamilyWorkflows: workflows, storepkg.FamilySkills: skills, storepkg.FamilyPlaybooks: playbooks, storepkg.FamilyProfiles: profiles, }, nil } // controlPlaneDeps serves the control plane and the per-org model answers // built on it. func controlPlaneDeps(deps *staterpc.Deps, control *controlplane.Server) { deps.ControlPlane = control deps.OrgModels = func(ctx context.Context, orgID string) (map[string]string, map[string]config.Provider, error) { models, err := control.ModelsFor(ctx, orgID) return models.Aliases, models.Providers, err } deps.OrgInstructions = control.InstructionsFor }