package connectors import ( "cmp" "context" "encoding/json/jsontext" json "encoding/json/v2" "errors" "fmt" "log/slog" "maps" "net/http" "regexp" "slices" "sort" "strings" "sync" "github.com/danielgtaylor/huma/v2" "github.com/jmoiron/sqlx" "github.com/samcharles93/archie-core/internal/platform/httpapi" "github.com/samcharles93/archie-core/internal/platform/seal" "github.com/samcharles93/archie-core/modules/connectors/internal/store" "github.com/samcharles93/archie-core/modules/governance" ) // validName is the shape of a connector name: it appears in logs and URLs. var validName = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`) // Module runs an organisation's accepted connectors, secret engines and guards, // keeps the credentials they are configured with, and serves their // administration. type Module struct { orgID string host *Host store registry sealer *seal.Sealer log *slog.Logger mu sync.RWMutex running map[string]*Connector engines map[string]*Engine guards map[string]*Guard // servers are the accepted MCP servers, and stops ends each one's // supervisor. servers map[string]*Server stops map[string]context.CancelFunc failed map[string]string // recheck, when set, re-checks saved configuration against a connector's // new declarations; an error refuses them. recheck Recheck } // Recheck re-checks saved configuration against the capabilities a connector // now declares. An error refuses the new declarations. type Recheck func(ctx context.Context, connector string, caps Capabilities) error // Use sets the hook called when a connector's declarations change. func (m *Module) Use(recheck Recheck) { m.mu.Lock() defer m.mu.Unlock() m.recheck = recheck } // registry is the module's persistence, so removal can be tested with a fake. type registry interface { List(ctx context.Context, orgID string) ([]store.Accepted, error) Upsert(ctx context.Context, a store.Accepted) error Delete(ctx context.Context, orgID, name string) (bool, error) Credential(ctx context.Context, orgID, name string) (store.Credential, bool, error) Credentials(ctx context.Context, orgID string) ([]store.Credential, error) PutCredential(ctx context.Context, c store.Credential) error SetCapabilities(ctx context.Context, orgID, name string, capabilities jsontext.Value) error Capabilities(ctx context.Context, orgID, name string) (jsontext.Value, bool, error) } // Sealing is the age identity credentials are sealed with, and the backup // recipients that can also open them. type Sealing struct { Identity string Backups []string } // NewModule returns the connectors module for orgID. func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*Module, error) { sealer, err := seal.New(sealing.Identity, sealing.Backups) if err != nil { return nil, err } return &Module{ orgID: orgID, host: NewHost(log), store: store.New(db), sealer: sealer, log: log, running: map[string]*Connector{}, engines: map[string]*Engine{}, guards: map[string]*Guard{}, failed: map[string]string{}, servers: map[string]*Server{}, stops: map[string]context.CancelFunc{}, }, nil } // Start launches every accepted secret engine, then every accepted connector, // since a connector's credentials may be resolved through an engine. One that // fails to start is reported by Ready and logged; the others still run. // // Its only error is the read of the accepted connectors, which happens before // anything launches, so a caller may call Start again after a failure without // launching anything twice. func (m *Module) Start(ctx context.Context) error { accepted, err := m.store.List(ctx, m.orgID) if err != nil { return err } for _, a := range ordered(accepted) { // A failure is recorded for Ready and logged; the rest still start. _ = m.launch(ctx, a) } return nil } // ordered starts secret engines first, then connectors, then MCP servers: a // credential may be resolved through an engine. func ordered(accepted []store.Accepted) []store.Accepted { out := slices.Clone(accepted) slices.SortStableFunc(out, func(a, b store.Accepted) int { return cmp.Compare(rank(a), rank(b)) }) return out } func rank(a store.Accepted) int { switch a.Contract { case store.ContractSecretEngine: return 0 case store.ContractMCP: return 2 } return 1 } // Get returns the running connector named name. An MCP server is reached // through Tool, since it answers lookups and actions rather than events. func (m *Module) Get(name string) (*Connector, bool) { m.mu.RLock() defer m.mu.RUnlock() c, ok := m.running[name] return c, ok } // Tool returns the connector or MCP server that declared name. func (m *Module) Tool(name string) (Tool, bool) { m.mu.RLock() defer m.mu.RUnlock() if c, ok := m.running[name]; ok { return c, true } s, ok := m.servers[name] return s, ok } // TargetFacts asks every running connector that serves target facts what it // knows about t, now, and keeps nothing. Connectors are asked in name order and // the first to supply a fact wins it. func (m *Module) TargetFacts(ctx context.Context, t Target) (TargetFacts, error) { m.mu.RLock() asked := make([]*Connector, 0, len(m.running)) for _, name := range slices.Sorted(maps.Keys(m.running)) { if c := m.running[name]; c.facts { asked = append(asked, c) } } m.mu.RUnlock() var out TargetFacts for _, c := range asked { got, err := c.TargetFacts(ctx, t) if err != nil { return TargetFacts{}, err } out.OwnerIdentity = cmp.Or(out.OwnerIdentity, got.OwnerIdentity) out.DataClass = cmp.Or(out.DataClass, got.DataClass) out.Count = cmp.Or(out.Count, got.Count) } return out, nil } // Guards lists the running guard extensions in name order. func (m *Module) Guards() []*Guard { m.mu.RLock() defer m.mu.RUnlock() var out []*Guard for _, name := range slices.Sorted(maps.Keys(m.guards)) { out = append(out, m.guards[name]) } return out } // Delivering lists the running connectors that send the named delivery, in // name order: the channels a message goes out on. func (m *Module) Delivering(delivery string) []*Connector { m.mu.RLock() defer m.mu.RUnlock() var out []*Connector for _, name := range slices.Sorted(maps.Keys(m.running)) { if c := m.running[name]; c.Delivers(delivery) { out = append(out, c) } } return out } // LinkableChannels lists the running connectors that can link a person's // channel identity to their account, in name order. func (m *Module) LinkableChannels(context.Context) ([]string, error) { m.mu.RLock() defer m.mu.RUnlock() var out []string for _, name := range slices.Sorted(maps.Keys(m.running)) { if m.running[name].LinksAccounts() { out = append(out, name) } } return out, nil } // LinkURL asks the running connector for the URL that delivers a link code. func (m *Module) LinkURL(ctx context.Context, connector, code string) (string, error) { c, ok := m.Get(connector) if !ok { return "", fmt.Errorf("connector %q isn't running", connector) } return c.LinkURL(ctx, code) } // Declarations lists every running connector's lookups and actions and every // MCP server's declared tools, ordered by name and then by tool name. func (m *Module) Declarations() []Declaration { m.mu.RLock() defer m.mu.RUnlock() var out []Declaration for _, name := range slices.Sorted(maps.Keys(m.running)) { out = append(out, m.running[name].Declarations()...) } for _, name := range slices.Sorted(maps.Keys(m.servers)) { out = append(out, m.servers[name].Declarations()...) } return out } // Ready reports an extension that failed to start or whose process has exited, // and an MCP server that doesn't answer. func (m *Module) Ready(context.Context) error { m.mu.RLock() var problems []string for name, reason := range m.failed { problems = append(problems, name+": "+reason) } for name := range m.running { if !m.host.Alive(name) { problems = append(problems, name+": process exited") } } for name := range m.engines { if !m.host.Alive(name) { problems = append(problems, name+": process exited") } } for name := range m.guards { if !m.host.Alive(name) { problems = append(problems, name+": process exited") } } servers := maps.Clone(m.servers) m.mu.RUnlock() for _, name := range slices.Sorted(maps.Keys(servers)) { if err := servers[name].Ready(); err != nil { problems = append(problems, name+": "+err.Error()) } } if len(problems) == 0 { return nil } sort.Strings(problems) return errors.New(strings.Join(problems, "; ")) } // Close stops every connector, secret engine and MCP server. func (m *Module) Close() { m.mu.Lock() stops, servers := m.stops, m.servers m.stops, m.servers = map[string]context.CancelFunc{}, map[string]*Server{} m.mu.Unlock() for _, stop := range stops { stop() } for _, s := range servers { s.Close() } m.host.Close() } // extensionKinds lists the API path and names for each kind of accepted // package. The group comes from the contract's entry in store.Contracts. func extensionKinds() []extensionKind { return []extensionKind{ {store.ContractConnector, "/api/connectors", "connector", "connector", "connectors", "Connectors"}, {store.ContractSecretEngine, "/api/secret-engines", "secret-engine", "secret engine", "secret engines", "Secret engines"}, {store.ContractGuard, "/api/guards", "guard", "guard", "guards", "Guards"}, {store.ContractMCP, "/api/mcp-servers", "mcp-server", "MCP server", "MCP servers", "MCP servers"}, } } type extensionKind struct { contract, path, noun, singular, plural, tag string } // ExtensionKind is one kind of accepted package and the group it belongs to. type ExtensionKind struct { Kind string `json:"kind" doc:"The API path segment that lists this kind"` Label string `json:"label"` Singular string `json:"singular"` Group string `json:"group" enum:"platform,integration,agent"` } func listExtensionKinds(context.Context, *struct{}) (*httpapi.Reply[[]ExtensionKind], error) { kinds := extensionKinds() out := make([]ExtensionKind, 0, len(kinds)) for _, k := range kinds { out = append(out, ExtensionKind{ Kind: strings.TrimPrefix(k.path, "/api/"), Label: k.tag, Singular: k.singular, Group: string(store.GroupOf(k.contract)), }) } return httpapi.OK(out), nil } // Register adds the API for accepted packages and credentials. func (m *Module) Register(api huma.API) { httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindConnector), huma.Operation{ OperationID: "list-extension-kinds", Method: http.MethodGet, Path: "/api/extension-kinds", Summary: "List the kinds of extension and the group each belongs to", Tags: []string{"Connectors"}, }, listExtensionKinds) for _, k := range extensionKinds() { tags := []string{k.tag} httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindConnector), huma.Operation{ OperationID: "list-" + k.noun + "s", Method: http.MethodGet, Path: k.path, Summary: "List accepted " + k.plural, Tags: tags, }, m.list(k.contract)) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionDelete, governance.KindConnector), huma.Operation{ OperationID: "remove-" + k.noun, Method: http.MethodDelete, Path: k.path + "/{name}", Summary: "Remove an accepted package", Tags: tags, }, m.remove(k.contract)) if k.contract == store.ContractMCP { httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindMCPServer), huma.Operation{ OperationID: "accept-mcp-server", Method: http.MethodPut, Path: k.path + "/{name}", Summary: "Accept an MCP server and the tools it may offer", Tags: tags, }, m.acceptServer) continue } httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindConnector), huma.Operation{ OperationID: "accept-" + k.noun, Method: http.MethodPut, Path: k.path + "/{name}", Summary: "Accept a package and start it", Tags: tags, }, m.accept(k.contract)) } tags := []string{"Credentials"} httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindSecret), huma.Operation{ OperationID: "list-credentials", Method: http.MethodGet, Path: "/api/credentials", Summary: "List credentials, never their values", Tags: tags, }, m.listCredentials) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindSecret), huma.Operation{ OperationID: "get-credential", Method: http.MethodGet, Path: "/api/credentials/{name}", Summary: "Get a credential, never its values", Tags: tags, }, m.getCredential) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionUpdate, governance.KindSecret), huma.Operation{ OperationID: "put-credential", Method: http.MethodPut, Path: "/api/credentials/{name}", Summary: "Store a credential sealed and restart what uses it", Tags: tags, }, m.putCredential) httpapi.Register(api, httpapi.AdminAuthorized(governance.ActionRead, governance.KindSecret), huma.Operation{ OperationID: "list-credential-kinds", Method: http.MethodGet, Path: "/api/credential-kinds", Summary: "List credential kinds and the fields each takes", Tags: tags, }, listCredentialKinds) } type acceptRequest struct { Path string `json:"path" doc:"The extension binary on the host"` SHA256 string `json:"sha256" doc:"The binary's digest; it runs only while the digest matches"` Egress []string `json:"egress,omitzero" doc:"Every host the extension may reach"` Env []string `json:"env,omitzero" doc:"Host environment variables the extension may see"` Settings map[string]string `json:"settings,omitzero"` // Credentials maps each credential slot the package declares to a // credential by name. Credentials map[string]string `json:"credentials,omitzero"` } func (m *Module) accept(contract string) func(context.Context, *packageInput) (*httpapi.Reply[packageState], error) { return func(ctx context.Context, in *packageInput) (*httpapi.Reply[packageState], error) { return m.acceptAs(ctx, in, contract) } } type packageInput struct { Name string `path:"name" doc:"Lowercase letters, digits and hyphens"` Body acceptRequest } type nameInput struct { Name string `path:"name"` } // packageState is an accepted package's name and state after a change. type packageState struct { Name string `json:"name"` State string `json:"state" enum:"running,removed"` } func (m *Module) acceptAs(ctx context.Context, in *packageInput, contract string) (*httpapi.Reply[packageState], error) { name, req := in.Name, in.Body if !validName.MatchString(name) || req.Path == "" || req.SHA256 == "" { return nil, huma.NewError(http.StatusBadRequest, "a connector needs a lowercase name, a path and a sha256") } settings, err := json.Marshal(req.Settings) if err != nil { return nil, huma.NewError(http.StatusBadRequest, err.Error()) } slots, err := json.Marshal(req.Credentials, json.FormatNilMapAsNull(false)) if err != nil { return nil, huma.NewError(http.StatusBadRequest, err.Error()) } a := store.Accepted{ OrgID: m.orgID, Name: name, Path: req.Path, SHA256: req.SHA256, Egress: req.Egress, Env: req.Env, Settings: settings, Contract: contract, Credentials: slots, // Only an MCP server declares its tools at acceptance; the column // holds a list either way. Tools: jsontext.Value(`[]`), } if err := m.store.Upsert(ctx, a); err != nil { return nil, huma.NewError(http.StatusInternalServerError, err.Error()) } if err := m.launch(context.WithoutCancel(ctx), a); err != nil { return nil, huma.NewError(http.StatusUnprocessableEntity, err.Error()) } // The extension declares its settings when it starts. A value it doesn't // declare, or a required one that is missing, is refused now rather than // left for the extension to misread. if err := m.checkAccepted(a); err != nil { _, _ = m.store.Delete(context.WithoutCancel(ctx), m.orgID, name) m.release(name) m.mu.Lock() delete(m.failed, name) m.mu.Unlock() return nil, huma.NewError(http.StatusUnprocessableEntity, err.Error()) } return httpapi.OK(packageState{Name: name, State: "running"}), nil } // launch starts or restarts one accepted package and records the outcome. A // name runs one package: whatever ran under it before is stopped first, so a // connector, a secret engine and an MCP server never share a name. func (m *Module) launch(ctx context.Context, a store.Accepted) error { m.release(a.Name) if a.Contract == store.ContractMCP { return m.startServer(ctx, a) } var settings map[string]string if len(a.Settings) > 0 { if err := json.Unmarshal(a.Settings, &settings); err != nil { return m.fail(ctx, a.Name, fmt.Errorf("settings: %w", err)) } } settings, secretSettings, err := m.splitSecretSettings(ctx, a, settings) if err != nil { return m.fail(ctx, a.Name, err) } secrets, err := m.secrets(ctx, a) if err != nil { return m.fail(ctx, a.Name, err) } maps.Copy(secrets, secretSettings) spec := Spec{ Name: a.Name, Path: a.Path, SHA256: a.SHA256, Env: a.Env, Egress: a.Egress, Settings: settings, Secrets: secrets, } if a.Contract == store.ContractSecretEngine { e, err := m.host.StartEngine(ctx, spec) if err != nil { return m.fail(ctx, a.Name, err) } m.mu.Lock() m.engines[a.Name] = e delete(m.failed, a.Name) m.mu.Unlock() m.log.InfoContext(ctx, "secret engine started", "engine", a.Name) return nil } if a.Contract == store.ContractGuard { g, err := m.host.StartGuard(ctx, spec) if err != nil { return m.fail(ctx, a.Name, err) } m.mu.Lock() m.guards[a.Name] = g delete(m.failed, a.Name) m.mu.Unlock() m.log.InfoContext(ctx, "guard started", "guard", a.Name) return nil } c, err := m.host.Start(ctx, spec) if err != nil { return m.fail(ctx, a.Name, err) } if err := m.checkDeclarations(ctx, a.Name, c.Capabilities()); err != nil { m.host.Stop(a.Name) return m.fail(ctx, a.Name, err) } if err := m.storeCapabilities(ctx, c); err != nil { m.host.Stop(a.Name) return m.fail(ctx, a.Name, err) } m.mu.Lock() m.running[a.Name] = c delete(m.failed, a.Name) m.mu.Unlock() m.log.InfoContext(ctx, "connector started", "connector", a.Name) return nil } // checkDeclarations refuses declarations core can't work with before they are // stored: a reserved delivery whose schema rejects what core sends, or saved // configuration that the new declarations break. The connector's previous // declarations stay in force. func (m *Module) checkDeclarations(ctx context.Context, name string, caps Capabilities) error { if err := checkCoreDeliveries(caps); err != nil { return err } m.mu.RLock() recheck := m.recheck m.mu.RUnlock() if recheck != nil { if err := recheck(ctx, name, caps); err != nil { return fmt.Errorf("saved configuration doesn't match the new declarations: %w", err) } } return nil } // storeCapabilities records what a connector declared, so workflows and // triggers can be checked against it while the connector is stopped. func (m *Module) storeCapabilities(ctx context.Context, c *Connector) error { raw, err := json.Marshal(c.Capabilities()) if err != nil { return fmt.Errorf("encode capabilities: %w", err) } return m.store.SetCapabilities(ctx, m.orgID, c.Name(), raw) } // refSuffix marks a setting that names a secret rather than holding one. The // host resolves it through the named engine, so the extension never reads the // reference. const refSuffix = "_ref" // splitSecretSettings separates a package's secret settings from the rest, // resolves each reference, and returns the plain settings and the secrets the // extension is configured with. A secret engine's own settings must hold // their values: an engine never waits on another engine. func (m *Module) splitSecretSettings(ctx context.Context, a store.Accepted, settings map[string]string) (map[string]string, map[string]string, error) { plain := make(map[string]string, len(settings)) secrets := map[string]string{} for key, value := range settings { name, isRef := strings.CutSuffix(key, refSuffix) if !isRef { plain[key] = value continue } if a.Contract == store.ContractSecretEngine { return nil, nil, fmt.Errorf("setting %q: a secret engine's own settings must hold their values", key) } engineName, ref, ok := strings.Cut(value, ":") if !ok || engineName == "" || ref == "" { return nil, nil, fmt.Errorf("setting %q wants engine:key", key) } m.mu.RLock() engine, running := m.engines[engineName] m.mu.RUnlock() if !running { return nil, nil, fmt.Errorf("setting %q: secret engine %q isn't running", key, engineName) } secret, err := engine.Resolve(ctx, ref) if err != nil { return nil, nil, fmt.Errorf("setting %q: %w", key, err) } secrets[name] = secret } return plain, secrets, nil } // release stops whatever ran under name, so one accepted package is what a // name runs. func (m *Module) release(name string) { m.mu.Lock() _, running := m.running[name] _, engine := m.engines[name] _, guard := m.guards[name] s, stop := m.servers[name], m.stops[name] delete(m.running, name) delete(m.engines, name) delete(m.guards, name) delete(m.servers, name) delete(m.stops, name) m.mu.Unlock() if running || engine || guard { m.host.Stop(name) } if stop != nil { stop() } if s != nil { s.Close() } } func (m *Module) fail(ctx context.Context, name string, err error) error { m.mu.Lock() delete(m.running, name) delete(m.engines, name) delete(m.guards, name) m.failed[name] = err.Error() m.mu.Unlock() m.log.ErrorContext(ctx, "extension failed to start", "extension", name, "error", err) return err }