Files
archie/modules/connectors/module.go
T

413 lines
12 KiB
Go

package connectors
import (
"cmp"
"context"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"log/slog"
"maps"
"net/http"
"regexp"
"slices"
"sort"
"strings"
"sync"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/internal/platform/httpjson"
"github.com/samcharles93/archie-core/modules/connectors/internal/seal"
"github.com/samcharles93/archie-core/modules/connectors/internal/store"
)
// validName is the shape of a connector name: it appears in logs and URLs.
var validName = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`)
// Module runs an organisation's accepted connectors, secret engines and guards,
// keeps the credentials they are configured with, and serves their
// administration.
type Module struct {
orgID string
host *Host
store registry
sealer *seal.Sealer
log *slog.Logger
mu sync.RWMutex
running map[string]*Connector
engines map[string]*Engine
guards map[string]*Guard
// servers are the accepted MCP servers, and stops ends each one's
// supervisor.
servers map[string]*Server
stops map[string]context.CancelFunc
failed map[string]string
}
// registry is the module's persistence, so removal can be tested with a fake.
type registry interface {
List(ctx context.Context, orgID string) ([]store.Accepted, error)
Upsert(ctx context.Context, a store.Accepted) error
Delete(ctx context.Context, orgID, name string) (bool, error)
Credential(ctx context.Context, orgID, name string) (store.Credential, bool, error)
Credentials(ctx context.Context, orgID string) ([]store.Credential, error)
PutCredential(ctx context.Context, c store.Credential) error
}
// Sealing is the age identity credentials are sealed with, and the backup
// recipients that can also open them.
type Sealing struct {
Identity string
Backups []string
}
// NewModule returns the connectors module for orgID.
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*Module, error) {
sealer, err := seal.New(sealing.Identity, sealing.Backups)
if err != nil {
return nil, err
}
return &Module{
orgID: orgID, host: NewHost(log), store: store.New(db), sealer: sealer, log: log,
running: map[string]*Connector{}, engines: map[string]*Engine{}, guards: map[string]*Guard{}, failed: map[string]string{},
servers: map[string]*Server{}, stops: map[string]context.CancelFunc{},
}, nil
}
// Start launches every accepted secret engine, then every accepted connector,
// since a connector's credentials may be resolved through an engine. One that
// fails to start is reported by Ready and logged; the others still run.
func (m *Module) Start(ctx context.Context) error {
accepted, err := m.store.List(ctx, m.orgID)
if err != nil {
return err
}
for _, a := range ordered(accepted) {
// A failure is recorded for Ready and logged; the rest still start.
_ = m.launch(ctx, a)
}
return nil
}
// ordered starts secret engines first, then connectors, then MCP servers: a
// credential may be resolved through an engine.
func ordered(accepted []store.Accepted) []store.Accepted {
out := slices.Clone(accepted)
slices.SortStableFunc(out, func(a, b store.Accepted) int {
return cmp.Compare(rank(a), rank(b))
})
return out
}
func rank(a store.Accepted) int {
switch a.Contract {
case store.ContractSecretEngine:
return 0
case store.ContractMCP:
return 2
}
return 1
}
// Get returns the running connector named name. An MCP server is reached
// through Tool, since it answers lookups and actions rather than events.
func (m *Module) Get(name string) (*Connector, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
c, ok := m.running[name]
return c, ok
}
// Tool returns the connector or MCP server that declared name.
func (m *Module) Tool(name string) (Tool, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
if c, ok := m.running[name]; ok {
return c, true
}
s, ok := m.servers[name]
return s, ok
}
// TargetFacts asks every running connector that serves target facts what it
// knows about t, now, and keeps nothing. Connectors are asked in name order and
// the first to supply a fact wins it.
func (m *Module) TargetFacts(ctx context.Context, t Target) (TargetFacts, error) {
m.mu.RLock()
asked := make([]*Connector, 0, len(m.running))
for _, name := range slices.Sorted(maps.Keys(m.running)) {
if c := m.running[name]; c.facts {
asked = append(asked, c)
}
}
m.mu.RUnlock()
var out TargetFacts
for _, c := range asked {
got, err := c.TargetFacts(ctx, t)
if err != nil {
return TargetFacts{}, err
}
out.OwnerIdentity = cmp.Or(out.OwnerIdentity, got.OwnerIdentity)
out.DataClass = cmp.Or(out.DataClass, got.DataClass)
out.Count = cmp.Or(out.Count, got.Count)
}
return out, nil
}
// Guards lists the running guard extensions in name order.
func (m *Module) Guards() []*Guard {
m.mu.RLock()
defer m.mu.RUnlock()
var out []*Guard
for _, name := range slices.Sorted(maps.Keys(m.guards)) {
out = append(out, m.guards[name])
}
return out
}
// Delivering lists the running connectors that send the named delivery, in
// name order: the channels a message goes out on.
func (m *Module) Delivering(delivery string) []*Connector {
m.mu.RLock()
defer m.mu.RUnlock()
var out []*Connector
for _, name := range slices.Sorted(maps.Keys(m.running)) {
if c := m.running[name]; c.Delivers(delivery) {
out = append(out, c)
}
}
return out
}
// Declarations lists every running connector's lookups and actions and every
// MCP server's declared tools, ordered by name and then by tool name.
func (m *Module) Declarations() []Declaration {
m.mu.RLock()
defer m.mu.RUnlock()
var out []Declaration
for _, name := range slices.Sorted(maps.Keys(m.running)) {
out = append(out, m.running[name].Declarations()...)
}
for _, name := range slices.Sorted(maps.Keys(m.servers)) {
out = append(out, m.servers[name].Declarations()...)
}
return out
}
// Ready reports an extension that failed to start or whose process has exited,
// and an MCP server that doesn't answer.
func (m *Module) Ready(context.Context) error {
m.mu.RLock()
var problems []string
for name, reason := range m.failed {
problems = append(problems, name+": "+reason)
}
for name := range m.running {
if !m.host.Alive(name) {
problems = append(problems, name+": process exited")
}
}
for name := range m.engines {
if !m.host.Alive(name) {
problems = append(problems, name+": process exited")
}
}
for name := range m.guards {
if !m.host.Alive(name) {
problems = append(problems, name+": process exited")
}
}
servers := maps.Clone(m.servers)
m.mu.RUnlock()
for _, name := range slices.Sorted(maps.Keys(servers)) {
if err := servers[name].Ready(); err != nil {
problems = append(problems, name+": "+err.Error())
}
}
if len(problems) == 0 {
return nil
}
sort.Strings(problems)
return errors.New(strings.Join(problems, "; "))
}
// Close stops every connector, secret engine and MCP server.
func (m *Module) Close() {
m.mu.Lock()
stops, servers := m.stops, m.servers
m.stops, m.servers = map[string]context.CancelFunc{}, map[string]*Server{}
m.mu.Unlock()
for _, stop := range stops {
stop()
}
for _, s := range servers {
s.Close()
}
m.host.Close()
}
// Routes adds the admin routes, each wrapped in guard.
func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) {
mux.Handle("PUT /api/connectors/{name}", guard(m.accept(store.ContractConnector)))
mux.Handle("PUT /api/secret-engines/{name}", guard(m.accept(store.ContractSecretEngine)))
mux.Handle("PUT /api/guards/{name}", guard(m.accept(store.ContractGuard)))
mux.Handle("PUT /api/mcp-servers/{name}", guard(http.HandlerFunc(m.acceptServer)))
mux.Handle("DELETE /api/connectors/{name}", guard(m.remove(store.ContractConnector)))
mux.Handle("DELETE /api/secret-engines/{name}", guard(m.remove(store.ContractSecretEngine)))
mux.Handle("DELETE /api/guards/{name}", guard(m.remove(store.ContractGuard)))
mux.Handle("DELETE /api/mcp-servers/{name}", guard(m.remove(store.ContractMCP)))
mux.Handle("PUT /api/credentials/{name}", guard(http.HandlerFunc(m.putCredential)))
mux.Handle("GET /api/credentials", guard(http.HandlerFunc(m.listCredentials)))
}
type acceptRequest struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Egress []string `json:"egress"`
Env []string `json:"env"`
Settings map[string]string `json:"settings"`
// Credentials maps each credential slot the package declares to a
// credential by name.
Credentials map[string]string `json:"credentials"`
}
func (m *Module) accept(contract string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { m.acceptAs(w, r, contract) }
}
func (m *Module) acceptAs(w http.ResponseWriter, r *http.Request, contract string) {
name := r.PathValue("name")
var req acceptRequest
if err := httpjson.Read(r, &req); err != nil {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
if !validName.MatchString(name) || req.Path == "" || req.SHA256 == "" {
httpjson.Error(w, http.StatusBadRequest, "a connector needs a lowercase name, a path and a sha256")
return
}
settings, err := json.Marshal(req.Settings)
if err != nil {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
slots, err := json.Marshal(req.Credentials, json.FormatNilMapAsNull(false))
if err != nil {
httpjson.Error(w, http.StatusBadRequest, err.Error())
return
}
a := store.Accepted{
OrgID: m.orgID, Name: name, Path: req.Path, SHA256: req.SHA256,
Egress: req.Egress, Env: req.Env, Settings: settings, Contract: contract, Credentials: slots,
// Only an MCP server declares its tools at acceptance; the column
// holds a list either way.
Tools: jsontext.Value(`[]`),
}
if err := m.store.Upsert(r.Context(), a); err != nil {
httpjson.Error(w, http.StatusInternalServerError, err.Error())
return
}
if err := m.launch(context.WithoutCancel(r.Context()), a); err != nil {
httpjson.Error(w, http.StatusUnprocessableEntity, err.Error())
return
}
httpjson.Write(w, http.StatusOK, map[string]string{"name": name, "state": "running"})
}
// launch starts or restarts one accepted package and records the outcome. A
// name runs one package: whatever ran under it before is stopped first, so a
// connector, a secret engine and an MCP server never share a name.
func (m *Module) launch(ctx context.Context, a store.Accepted) error {
m.release(a.Name)
if a.Contract == store.ContractMCP {
return m.startServer(ctx, a)
}
var settings map[string]string
if len(a.Settings) > 0 {
if err := json.Unmarshal(a.Settings, &settings); err != nil {
return m.fail(ctx, a.Name, fmt.Errorf("settings: %w", err))
}
}
secrets, err := m.secrets(ctx, a)
if err != nil {
return m.fail(ctx, a.Name, err)
}
spec := Spec{
Name: a.Name, Path: a.Path, SHA256: a.SHA256,
Env: a.Env, Egress: a.Egress, Settings: settings, Secrets: secrets,
}
if a.Contract == store.ContractSecretEngine {
e, err := m.host.StartEngine(ctx, spec)
if err != nil {
return m.fail(ctx, a.Name, err)
}
m.mu.Lock()
m.engines[a.Name] = e
delete(m.failed, a.Name)
m.mu.Unlock()
m.log.InfoContext(ctx, "secret engine started", "engine", a.Name)
return nil
}
if a.Contract == store.ContractGuard {
g, err := m.host.StartGuard(ctx, spec)
if err != nil {
return m.fail(ctx, a.Name, err)
}
m.mu.Lock()
m.guards[a.Name] = g
delete(m.failed, a.Name)
m.mu.Unlock()
m.log.InfoContext(ctx, "guard started", "guard", a.Name)
return nil
}
c, err := m.host.Start(ctx, spec)
if err != nil {
return m.fail(ctx, a.Name, err)
}
m.mu.Lock()
m.running[a.Name] = c
delete(m.failed, a.Name)
m.mu.Unlock()
m.log.InfoContext(ctx, "connector started", "connector", a.Name)
return nil
}
// release stops whatever ran under name, so one accepted package is what a
// name runs.
func (m *Module) release(name string) {
m.mu.Lock()
_, running := m.running[name]
_, engine := m.engines[name]
_, guard := m.guards[name]
s, stop := m.servers[name], m.stops[name]
delete(m.running, name)
delete(m.engines, name)
delete(m.guards, name)
delete(m.servers, name)
delete(m.stops, name)
m.mu.Unlock()
if running || engine || guard {
m.host.Stop(name)
}
if stop != nil {
stop()
}
if s != nil {
s.Close()
}
}
func (m *Module) fail(ctx context.Context, name string, err error) error {
m.mu.Lock()
delete(m.running, name)
delete(m.engines, name)
delete(m.guards, name)
m.failed[name] = err.Error()
m.mu.Unlock()
m.log.ErrorContext(ctx, "extension failed to start", "extension", name, "error", err)
return err
}