Files
archie/internal/app/statestore/server.go
T
sam f92c5b4536
ci/woodpecker/pr/quality Pipeline was successful
Quality / check (pull_request) Canceled after 7m39s
chore(runtime): delete Kit and external agent harness support (#1877)
Archie runs its own agent loop only. Removes Kit resolution and launch,
the harness runner and adapters, the setup terminal and sandbox view,
harness secrets and OAuth capture (table dropped in migration 0075), the
task-tools contract that only Kit containers reached, and the
sandbox-kit-spec dependency. Agent profiles carry image and tools only.
The egress proxy keeps the model-session path; the sandbox relay code
stays for #1867.
2026-11-12 21:29:05 +11:00

565 lines
23 KiB
Go

// state_store.go composes the standalone archie-state-store process. It mirrors
// gateway.go / cmd/archie-gateway: the binary is a thin main that passes
// process inputs into Run, which serves the task and event-capture
// stores from Postgres, registers every StateStore gRPC handler (the same service .4.2 serves
// in-process on the daemon), and shuts down cleanly on ctx cancellation.
package statestore
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"github.com/jackc/pgx/v5/pgxpool"
"google.golang.org/grpc"
"github.com/samcharles93/archie-core/internal/app/servicekit"
"github.com/samcharles93/archie-core/internal/config"
"github.com/samcharles93/archie-core/internal/domain/access"
"github.com/samcharles93/archie-core/internal/domain/alert"
"github.com/samcharles93/archie-core/internal/domain/audit"
"github.com/samcharles93/archie-core/internal/domain/health"
"github.com/samcharles93/archie-core/internal/domain/identity"
"github.com/samcharles93/archie-core/internal/domain/org"
"github.com/samcharles93/archie-core/internal/domain/presence"
"github.com/samcharles93/archie-core/internal/domain/storecontract"
"github.com/samcharles93/archie-core/internal/domain/storepkg"
"github.com/samcharles93/archie-core/internal/domain/usage"
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanestore"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
"github.com/samcharles93/archie-core/internal/infrastructure/readiness"
"github.com/samcharles93/archie-core/internal/infrastructure/rpcidentity"
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
registry "github.com/samcharles93/archie-core/internal/infrastructure/storepkg"
"github.com/samcharles93/archie-core/internal/infrastructure/workflowsteps"
"github.com/samcharles93/archie-core/internal/secret"
)
// Options contains process inputs for the standalone State Store.
type Options struct {
Config string
Overlay string
// Listen is the gRPC listen address. Without [tls] it must be loopback.
Listen string
// ReadyAddr, when non-empty, starts an HTTP readiness surface on the
// address: GET /healthz (liveness) and GET /health/detailed (the
// state_db probe). Empty disables it.
ReadyAddr string
}
// Run serves the task and event-capture stores from Postgres,
// registers every StateStore gRPC handler, and serves until ctx is cancelled.
// The conversation store belongs to the separate archie-gateway process. The
// store service owns its own DB lifecycle, so
// b.cleanup() is the sole owner closing b.st here (in-process owner).
func Run(ctx context.Context, options Options) error { //nolint:cyclop // the composition root's setup sequence is deliberately flat and sequential
b := newServer()
defer b.cleanup()
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
return err
}
listen, err := servicekit.ResolveListen("state", options.Listen, b.cfg.Services.Get(config.ServiceNameState).Listen)
if err != nil {
return err
}
b.log = b.log.With("component", "state-store", "listen", listen)
if err := b.openStateStore(ctx); err != nil {
return err
}
identityStore, ok := b.st.(interface {
BootstrapIdentities(context.Context, []string) error
})
if !ok {
return fmt.Errorf("state store does not support identity bootstrap")
}
legacyNames := servicekit.IdentityNames(b.cfg)
if err := identityStore.BootstrapIdentities(ctx, legacyNames); err != nil {
return fmt.Errorf("bootstrap identities: %w", err)
}
// The resumable default org/workspace upgrade runs before this process
// serves anything: the State Store refuses scoped calls until its upgrade
// has finished, rather than serving records with no org.
// A store
// without the upgrader is one that owns no tenant boundary yet, so the
// absence degrades to "no upgrade" rather than aborting boot.
if upgrader, ok := b.st.(org.Upgrader); ok {
if err := upgrader.UpgradeDefaultOrg(ctx); err != nil {
return fmt.Errorf("upgrade default org: %w", err)
}
}
if err := b.seedAndValidatePolicies(ctx); err != nil {
return err
}
// The validating side's control plane, built here at the composition root
// before the first definition is read or replaced.
resources, ok := b.st.(controlplanestore.ResourceStore)
if !ok {
return fmt.Errorf("state store does not support control-plane resources")
}
control, err := openStateStoreControlPlane(resources)
if err != nil {
return err
}
_, skipped, err := control.ImportConfig(ctx, b.cfg)
if err != nil {
return fmt.Errorf("import control-plane resources: %w", err)
}
b.reportUnseededResources(skipped)
grants, err := b.runCredentials(ctx)
if err != nil {
return err
}
//nolint:contextcheck // grpc.StreamServerInterceptor has no context.Context parameter; TaskGrants.StreamInterceptor derives its context from stream.Context() instead
opts, err := stateStoreServerOpts(listen, b.cfg.TLS, grants, b.callers())
if err != nil {
return err
}
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", listen)
if err != nil {
return fmt.Errorf("listen for state store: %w", err)
}
defer listener.Close()
b.log.InfoContext(ctx, "archie-state-store running", "addr", listener.Addr().String(), "tls", b.cfg.TLS.CertDir != "")
if err := b.startOptionalSurfaces(ctx, options); err != nil {
return err
}
// Boot is over and every listener this process serves is bound: the gRPC
// contract below and, when -ready-addr asked for one, the readiness HTTP
// surface. systemd's READY=1 asserts the same fact, so the announcement
// goes out here rather than from a second notion of "started".
// Not earlier: a readiness surface that fails to bind
// fails the boot, and READY must not precede it. Not later: Serve begins
// accepting the moment it is called.
b.announceReady()
deps := b.stateStoreDeps(grants)
controlPlaneDeps(&deps, control)
b.startSweeps(ctx, deps)
return serveStateStore(ctx, listener, deps, opts)
}
// seedAndValidatePolicies seeds the shipped role policies for every org that
// lacks them, grants the agent identities their org's shipped developer role,
// and re-validates every stored policy this process serves. An invalid
// policy at any level keeps denying its level and is named as a problem
// by the engines the consumers build; the alert sweep raises an alert for it,
// so boot goes on and the dashboard can repair it. A store without the access
// surfaces degrades: there is no chain to seed.
func (b *server) seedAndValidatePolicies(ctx context.Context) error {
policies, ok := b.st.(access.PolicyStore)
if !ok {
return nil
}
if agents, ok := b.st.(interface {
EnsureAgentOrgMembership(context.Context) error
}); ok {
if err := agents.EnsureAgentOrgMembership(ctx); err != nil {
return fmt.Errorf("grant agent org membership: %w", err)
}
}
if orgs, ok := b.st.(org.Repository); ok {
list, err := orgs.ListOrgs(ctx)
if err != nil {
return fmt.Errorf("list orgs for policy seeding: %w", err)
}
for _, o := range list {
if err := policies.EnsureShippedOrgPolicies(ctx, o.ID); err != nil {
return fmt.Errorf("seed shipped org policies for %s: %w", o.ID, err)
}
}
}
stored, err := policies.ListPolicies(ctx)
if err != nil {
return fmt.Errorf("load stored policies: %w", err)
}
engine := infraaccess.New(stored)
for _, problem := range engine.Problems() {
b.log.ErrorContext(ctx, "stored access policy is invalid and denies its level",
"policy", problem.Policy.ID, "level", problem.Policy.Level, "err", problem.Err)
}
return nil
}
// reportUnseededResources logs every kind ImportConfig could not seed.
//
// Reported, not fatal: a stale TOML value in a setting the database owns must
// not stop this process from starting. The kind stays ABSENT rather than
// half-written, so the file document's value is the one in effect for every
// reader -- the layering leaves a kind with no stored value alone
// (controlplane.resourceReader) -- and the operator's fix is still the file.
// Whether the value may RUN is decided where it is used: boot.runtimeConfig
// validates the effective document and refuses to start archied or the Gateway
// with one it cannot run. The seed is retried on this process's next start, so
// a corrected config.toml takes effect when archie-state-store restarts.
//
// It is a method rather than a loop in Run because the boot sequence
// is at its complexity budget: the branch belongs to the report's own policy,
// not to the sequence that triggers it.
func (b *server) reportUnseededResources(skipped []controlplanestore.SeedSkip) {
for _, skip := range skipped {
if len(skip.Removed) > 0 {
b.log.Warn("removed stored entries that no longer validate; their earlier revision is in the resource history",
"kind", skip.Kind, "removed", skip.Removed)
continue
}
b.log.Error("control-plane resource not seeded; the kind stays absent and the file's value is the one in effect",
"kind", skip.Kind, "err", skip.Err)
}
}
// openStateStoreControlPlane builds the State Store's control plane server: the
// validating side of the workflow step vocabulary, registered here at the
// composition root before the first definition is read or replaced. The same
// provider set (internal/infrastructure/workflowsteps) is what archie-agent
// registers before it compiles one, so within one build a definition this
// server admits is compilable there; a skewed deploy is only fixed by a
// matching deploy.
func openStateStoreControlPlane(resources controlplanestore.ResourceStore) (*controlplanestore.Server, error) {
steps, err := workflowsteps.NewManager()
if err != nil {
return nil, fmt.Errorf("register workflow step vocabulary: %w", err)
}
server, err := controlplanestore.NewServer(resources, steps)
if err != nil {
return nil, fmt.Errorf("build control plane server: %w", err)
}
return server, nil
}
// openStateStore composes the State Store process's persistence: it resolves
// secrets (for the binding cipher), opens and migrates the process-scoped
// PostgreSQL pool (fail closed), and serves the task and event-capture stores
// from it.
// It does NOT open gateway chat sessions -- those live in the separate
// archie-gateway process on their own store and are out of state-store scope.
func (b *server) openStateStore(ctx context.Context) error {
cfg, log := b.cfg, b.log
// The State Store serves the installed packages extension engines come from,
// so it resolves its own references through the env engine only.
secrets := secret.NewRegistry()
b.secrets = secrets
bindingCipher, err := bindingCipherFromConfig(cfg, secrets)
if err != nil {
log.ErrorContext(ctx, "configure bindings cipher", "err", err)
return err
}
// Open the PostgreSQL pool and apply the schema before anything else: the
// State Store fails closed without a working database_url.
if err := b.openStateStorePool(ctx); err != nil {
return err
}
b.openTaskStore()
b.openEDAStore(bindingCipher)
return nil
}
// stateStoreDeps assembles the store surfaces the StateStore service fronts.
// b.st is the narrow storecontract.TaskStore; the concrete store also implements
// the capture/mapping/binding surfaces, so each is asserted here (the same
// pattern the daemon's wireWebStoreSurfaces uses) and a store that lacks one
// degrades that group rather than aborting boot.
func (b *server) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
deps := staterpc.Deps{
LogFeed: b.logFeed,
Tasks: b.st, Log: b.log, Grants: grants,
}
if b.pg != nil {
packages := storepkg.Service{
Registry: registry.OCIRegistry{},
Store: postgres.NewInstalledPackages(b.pg),
Catalogue: b.liveCatalogue(),
}
// A projection failure degrades the org resources a package would have
// contributed to, the way a group above degrades: the State Store boots
// and reports, rather than refusing to serve the task tables.
projections, err := newPackageProjections(b.pg)
if err != nil {
b.log.Error("build package projections", "err", err)
} else {
packages.Projections = projections
}
deps.Packages = packages
}
if identities, ok := b.st.(identity.Repository); ok {
deps.Identities = identities
}
// Subject bindings are asserted separately from the repository facade: only
// the authenticating path resolves a verified subject, and a store that
// cannot is one where the dashboard cannot name its caller rather than one
// that fails to boot.
if bindings, ok := b.st.(identity.SubjectBinding); ok {
deps.SubjectBindings = bindings
}
if tokens, ok := b.st.(identity.PersonalTokenStore); ok {
deps.PersonalTokens = tokens
}
// Task logs live in the state directory, which this process owns, and the
// dashboard process owns no such directory -- so this is where a task-log
// read is served from. The reader is
// the daemon's own registry over the configured state_dir.
//
// The nil check is on the registry, not on the interface it is assigned
// to: a nil *logging.TaskRegistry stored in this field produces a non-nil
// interface holding a nil pointer, which the server's own
// `deps.TaskLogs == nil` guard would not catch. Leaving the field unset is
// what makes the RPC answer Unavailable -- the honest "this process cannot
// read logs" -- instead of depending on nil-receiver behaviour to notice.
if b.taskLogs != nil {
deps.TaskLogs = orgTaskLogs{TaskLogStore: b.taskLogs, tasks: b.st}
}
// The event-capture contracts are served by the event-capture store, not
// the task store. BindingTaskCreator stays below on the
// task store, because creating a task is the one thing it still does.
if b.eda != nil {
deps.Captures = b.eda
deps.Refusals = b.eda
deps.Mappings = b.eda
deps.Bindings = b.eda
deps.BindingDispatcher = b.eda
deps.PlaybookDispatcher = b.eda
deps.EventTypes = b.eda
deps.Sources = b.eda
deps.MappingMatches = b.eda
}
// tool_call events project into the tool_calls collection on the same
// event-capture store: this process legitimately owns both, so the
// projection rides on the task-store surface it decorates. With no
// event-capture store (or no task store) there is nothing to project
// through, and Tasks passes through unwrapped rather than decorating a
// nil operand.
if b.eda != nil && b.st != nil {
deps.Tasks = newToolCallProjectingTaskStore(b.st, b.eda, b.log)
}
if btc, ok := b.st.(storecontract.BindingTaskCreator); ok {
deps.BindingTaskCreator = btc
}
b.executionDeps(&deps)
b.accessDeps(&deps)
b.scheduleDeps(&deps)
if css, ok := b.st.(storecontract.ConfigSnapshotStore); ok {
deps.ConfigSnapshots = css
}
if ps, ok := b.st.(storecontract.PresenceStore); ok {
deps.Presence = ps
}
if as, ok := b.st.(storecontract.ApplyStatusStore); ok {
deps.ApplyStatus = as
}
if cs, ok := b.st.(storecontract.ChannelStatusStore); ok {
deps.ChannelStatus = cs
}
return deps
}
// executionDeps wires the execution-tree surfaces (workflow calls, step
// recording/reading, cancellation) from the same store, degrading each one
// independently the store lacks it rather than failing the boot.
func (b *server) executionDeps(deps *staterpc.Deps) {
if wc, ok := b.st.(storecontract.WorkflowCaller); ok {
deps.WorkflowCalls = wc
}
if sr, ok := b.st.(storecontract.StepRecorder); ok {
deps.Steps = sr
}
if ec, ok := b.st.(storecontract.ExecutionCanceller); ok {
deps.Canceller = ec
}
if ur, ok := b.st.(usage.Recorder); ok {
deps.Usage = ur
}
if ut, ok := b.st.(storecontract.UsageStore); ok {
deps.UsageTotals = ut
}
if sr, ok := b.st.(storecontract.StepReader); ok {
deps.StepReader = sr
}
}
// runCredentials builds the run-credential check over this store and drops
// the credentials that expired while nothing revoked them.
func (b *server) runCredentials(ctx context.Context) (*staterpc.TaskGrants, error) {
store, ok := b.st.(storecontract.RunCredentialStore)
if !ok {
return nil, fmt.Errorf("state store does not support run credentials")
}
grants := &staterpc.TaskGrants{Store: store, Log: b.log}
if err := grants.Sweep(ctx); err != nil {
return nil, fmt.Errorf("sweep expired run credentials: %w", err)
}
return grants, nil
}
// callers resolves the service and principal of each call against this
// store's principals.
func (b *server) callers() rpcidentity.Callers {
ps, _ := b.st.(access.PrincipalSource)
workspaces, _ := b.st.(access.WorkspaceLister)
return rpcidentity.Callers{Principals: ps, Workspaces: workspaces}
}
// accessDeps wires the policy chain and its denial records from the same
// store. A store without them -- one that owns
// no tenant boundary yet -- degrades the access RPCs rather than failing the
// boot, the same pattern the other optional surfaces use.
func (b *server) accessDeps(deps *staterpc.Deps) {
if accounts, ok := b.st.(identity.Accounts); ok {
deps.Accounts = accounts
}
if ps, ok := b.st.(access.PrincipalSource); ok {
deps.Principals = ps
}
if ps, ok := b.st.(access.PolicyStore); ok {
deps.Policies = ps
if owners, ok := b.st.(ownerDirectory); ok {
deps.Policies = guardedPolicies{PolicyStore: ps, owners: owners}
}
}
if ds, ok := b.st.(access.DenialStore); ok {
deps.Denials = ds
}
if os, ok := b.st.(org.Repository); ok {
deps.Orgs = os
}
if as, ok := b.st.(alert.Store); ok {
deps.Alerts = as
}
if feed, ok := b.st.(audit.Store); ok {
deps.Audit = feed
}
}
// serveStateStore registers the StateStore gRPC service and serves it until
// ctx ends.
func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.Deps, opts []grpc.ServerOption) error {
server := grpc.NewServer(append(servicekit.ServerLimits(), opts...)...)
staterpc.RegisterServer(server, deps)
return servicekit.Serve(ctx, server, listener)
}
// startStateStoreReadiness starts a minimal HTTP readiness surface on
// readyAddr: GET /healthz and /health answer 200 as a liveness probe (the
// process is up), while GET /health/detailed runs the state_db readiness probe
// and answers 503 when the store is degraded. It is the standalone process's
// counterpart to the daemon's /healthz + /health/detailed (internal/webui).
func (b *server) startStateStoreReadiness(ctx context.Context, readyAddr string) error {
registry := b.readinessRegistry()
mux := http.NewServeMux()
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
mux.HandleFunc("GET /health/detailed", func(w http.ResponseWriter, r *http.Request) {
report := registry.Run(r.Context())
w.Header().Set("Content-Type", "application/json")
if report.Status != health.StatusOK {
w.WriteHeader(http.StatusServiceUnavailable)
}
_ = json.NewEncoder(w).Encode(report)
})
return b.serveHealth(ctx, readyAddr, mux, "state store readiness")
}
func (b *server) readinessRegistry() *health.Registry {
return health.NewRegistry(readiness.NewStoreProbe(b.st))
}
// startOptionalSurfaces starts the HTTP surfaces this process serves beside
// the gRPC contract, each enabled only by its own flag.
//
// They are gathered here rather than branched inline because Run is
// at its complexity budget: a surface that is off by default belongs to its
// own decision, not to the boot sequence that triggers it -- the same reason
// reportUnseededResources is a method.
func (b *server) startOptionalSurfaces(ctx context.Context, options Options) error {
if ps, ok := b.st.(storecontract.PresenceStore); ok {
go presence.Run(ctx, presence.StateStore, servicekit.Build(), ps, b.readinessRegistry(), b.log)
}
if options.ReadyAddr != "" {
if err := b.startStateStoreReadiness(ctx, options.ReadyAddr); err != nil {
return err
}
}
return nil
}
// stateStoreServerOpts serves listen with this process's [tls] certificate. A client
// certificate is optional here alone: a task container has none and is
// confined by its grant. Authentication runs before attribution.
//
// The keepalive enforcement policy matches staterpc.Dial's clients, whose
// idle watches grpc's default policy would answer with GOAWAY
// too_many_pings. See staterpc.ServerKeepaliveOption.
func stateStoreServerOpts(listen string, tls config.TLS, grants *staterpc.TaskGrants, callers rpcidentity.Callers) ([]grpc.ServerOption, error) {
id, err := servicekit.Identity(tls, presence.StateStore)
if err != nil {
return nil, err
}
creds, err := id.ServerOption(listen, true)
if err != nil {
return nil, fmt.Errorf("state store: %w", err)
}
return []grpc.ServerOption{
creds,
staterpc.ServerKeepaliveOption(),
grpc.ChainUnaryInterceptor(grants.UnaryInterceptor(), callers.Unary()),
grpc.ChainStreamInterceptor(grants.StreamInterceptor(), callers.Stream()),
}, nil
}
// newPackageProjections builds the family projectors an installed package's
// contributions apply through: workflows, playbooks, skills and agent profiles become
// the org resources the runtime reads live. The step vocabulary
// (stepVocabulary) is the same one this process's control plane validates
// dashboard edits with, so a definition the projector admits is compilable
// where it dispatches (openStateStoreControlPlane for the pairing).
func newPackageProjections(pool *pgxpool.Pool) (map[string]storepkg.FamilyProjector, error) {
steps, err := workflowsteps.NewManager()
if err != nil {
return nil, fmt.Errorf("register workflow step vocabulary: %w", err)
}
workflows, err := controlplanestore.NewWorkflowPackageProjector(
postgres.NewResources(pool), postgres.NewPackageContributions(pool), steps)
if err != nil {
return nil, fmt.Errorf("build workflow package projector: %w", err)
}
resources, ledger := postgres.NewResources(pool), postgres.NewPackageContributions(pool)
playbooks, err := controlplanestore.NewPlaybookPackageProjector(resources, ledger)
if err != nil {
return nil, fmt.Errorf("build playbook package projector: %w", err)
}
profiles, err := controlplanestore.NewProfilePackageProjector(resources, ledger)
if err != nil {
return nil, fmt.Errorf("build profile package projector: %w", err)
}
skills, err := controlplanestore.NewSkillPackageProjector(resources, ledger)
if err != nil {
return nil, fmt.Errorf("build skill package projector: %w", err)
}
return map[string]storepkg.FamilyProjector{
storepkg.FamilyWorkflows: workflows,
storepkg.FamilySkills: skills,
storepkg.FamilyPlaybooks: playbooks,
storepkg.FamilyProfiles: profiles,
}, nil
}
// controlPlaneDeps serves the control plane and the per-org model answers
// built on it.
func controlPlaneDeps(deps *staterpc.Deps, control *controlplanestore.Server) {
deps.ControlPlane = control
deps.OrgModels = func(ctx context.Context, orgID string) (map[string]string, map[string]config.Provider, error) {
models, err := control.ModelsFor(ctx, orgID)
return models.Aliases, models.Providers, err
}
deps.OrgInstructions = control.InstructionsFor
}