354 lines
11 KiB
Go
354 lines
11 KiB
Go
package connectors
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"encoding/json/jsontext"
|
|
json "encoding/json/v2"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"maps"
|
|
"net/http"
|
|
"regexp"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"github.com/samcharles93/archie-core/internal/platform/httpjson"
|
|
"github.com/samcharles93/archie-core/modules/connectors/internal/seal"
|
|
"github.com/samcharles93/archie-core/modules/connectors/internal/store"
|
|
)
|
|
|
|
// validName is the shape of a connector name: it appears in logs and URLs.
|
|
var validName = regexp.MustCompile(`^[a-z][a-z0-9-]{0,62}$`)
|
|
|
|
// Module runs an organisation's accepted connectors and secret engines,
|
|
// keeps the credentials they are configured with, and serves their
|
|
// administration.
|
|
type Module struct {
|
|
orgID string
|
|
host *Host
|
|
store registry
|
|
sealer *seal.Sealer
|
|
log *slog.Logger
|
|
|
|
mu sync.RWMutex
|
|
running map[string]*Connector
|
|
engines map[string]*Engine
|
|
// servers are the accepted MCP servers, and stops ends each one's
|
|
// supervisor.
|
|
servers map[string]*Server
|
|
stops map[string]context.CancelFunc
|
|
failed map[string]string
|
|
}
|
|
|
|
// registry is the module's persistence, so removal can be tested with a fake.
|
|
type registry interface {
|
|
List(ctx context.Context, orgID string) ([]store.Accepted, error)
|
|
Upsert(ctx context.Context, a store.Accepted) error
|
|
Delete(ctx context.Context, orgID, name string) (bool, error)
|
|
Credential(ctx context.Context, orgID, name string) (store.Credential, bool, error)
|
|
Credentials(ctx context.Context, orgID string) ([]store.Credential, error)
|
|
PutCredential(ctx context.Context, c store.Credential) error
|
|
}
|
|
|
|
// Sealing is the age identity credentials are sealed with, and the backup
|
|
// recipients that can also open them.
|
|
type Sealing struct {
|
|
Identity string
|
|
Backups []string
|
|
}
|
|
|
|
// NewModule returns the connectors module for orgID.
|
|
func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, sealing Sealing) (*Module, error) {
|
|
sealer, err := seal.New(sealing.Identity, sealing.Backups)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Module{
|
|
orgID: orgID, host: NewHost(log), store: store.New(db), sealer: sealer, log: log,
|
|
running: map[string]*Connector{}, engines: map[string]*Engine{}, failed: map[string]string{},
|
|
servers: map[string]*Server{}, stops: map[string]context.CancelFunc{},
|
|
}, nil
|
|
}
|
|
|
|
// Start launches every accepted secret engine, then every accepted connector,
|
|
// since a connector's credentials may be resolved through an engine. One that
|
|
// fails to start is reported by Ready and logged; the others still run.
|
|
func (m *Module) Start(ctx context.Context) error {
|
|
accepted, err := m.store.List(ctx, m.orgID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, a := range ordered(accepted) {
|
|
// A failure is recorded for Ready and logged; the rest still start.
|
|
_ = m.launch(ctx, a)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ordered starts secret engines first, then connectors, then MCP servers: a
|
|
// credential may be resolved through an engine.
|
|
func ordered(accepted []store.Accepted) []store.Accepted {
|
|
out := slices.Clone(accepted)
|
|
slices.SortStableFunc(out, func(a, b store.Accepted) int {
|
|
return cmp.Compare(rank(a), rank(b))
|
|
})
|
|
return out
|
|
}
|
|
|
|
func rank(a store.Accepted) int {
|
|
switch a.Contract {
|
|
case store.ContractSecretEngine:
|
|
return 0
|
|
case store.ContractMCP:
|
|
return 2
|
|
}
|
|
return 1
|
|
}
|
|
|
|
// Get returns the running connector named name. An MCP server is reached
|
|
// through Tool, since it answers lookups and actions rather than events.
|
|
func (m *Module) Get(name string) (*Connector, bool) {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
c, ok := m.running[name]
|
|
return c, ok
|
|
}
|
|
|
|
// Tool returns the connector or MCP server that declared name.
|
|
func (m *Module) Tool(name string) (Tool, bool) {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
if c, ok := m.running[name]; ok {
|
|
return c, true
|
|
}
|
|
s, ok := m.servers[name]
|
|
return s, ok
|
|
}
|
|
|
|
// Delivering lists the running connectors that send the named delivery, in
|
|
// name order: the channels a message goes out on.
|
|
func (m *Module) Delivering(delivery string) []*Connector {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
var out []*Connector
|
|
for _, name := range slices.Sorted(maps.Keys(m.running)) {
|
|
if c := m.running[name]; c.Delivers(delivery) {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Declarations lists every running connector's lookups and actions and every
|
|
// MCP server's declared tools, ordered by name and then by tool name.
|
|
func (m *Module) Declarations() []Declaration {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
var out []Declaration
|
|
for _, name := range slices.Sorted(maps.Keys(m.running)) {
|
|
out = append(out, m.running[name].Declarations()...)
|
|
}
|
|
for _, name := range slices.Sorted(maps.Keys(m.servers)) {
|
|
out = append(out, m.servers[name].Declarations()...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Ready reports an extension that failed to start or whose process has exited,
|
|
// and an MCP server that doesn't answer.
|
|
func (m *Module) Ready(context.Context) error {
|
|
m.mu.RLock()
|
|
var problems []string
|
|
for name, reason := range m.failed {
|
|
problems = append(problems, name+": "+reason)
|
|
}
|
|
for name := range m.running {
|
|
if !m.host.Alive(name) {
|
|
problems = append(problems, name+": process exited")
|
|
}
|
|
}
|
|
for name := range m.engines {
|
|
if !m.host.Alive(name) {
|
|
problems = append(problems, name+": process exited")
|
|
}
|
|
}
|
|
servers := maps.Clone(m.servers)
|
|
m.mu.RUnlock()
|
|
for _, name := range slices.Sorted(maps.Keys(servers)) {
|
|
if err := servers[name].Ready(); err != nil {
|
|
problems = append(problems, name+": "+err.Error())
|
|
}
|
|
}
|
|
if len(problems) == 0 {
|
|
return nil
|
|
}
|
|
sort.Strings(problems)
|
|
return errors.New(strings.Join(problems, "; "))
|
|
}
|
|
|
|
// Close stops every connector, secret engine and MCP server.
|
|
func (m *Module) Close() {
|
|
m.mu.Lock()
|
|
stops, servers := m.stops, m.servers
|
|
m.stops, m.servers = map[string]context.CancelFunc{}, map[string]*Server{}
|
|
m.mu.Unlock()
|
|
for _, stop := range stops {
|
|
stop()
|
|
}
|
|
for _, s := range servers {
|
|
s.Close()
|
|
}
|
|
m.host.Close()
|
|
}
|
|
|
|
// Routes adds the admin routes, each wrapped in guard.
|
|
func (m *Module) Routes(mux *http.ServeMux, guard func(http.Handler) http.Handler) {
|
|
mux.Handle("PUT /api/connectors/{name}", guard(m.accept(store.ContractConnector)))
|
|
mux.Handle("PUT /api/secret-engines/{name}", guard(m.accept(store.ContractSecretEngine)))
|
|
mux.Handle("PUT /api/mcp-servers/{name}", guard(http.HandlerFunc(m.acceptServer)))
|
|
mux.Handle("DELETE /api/connectors/{name}", guard(m.remove(store.ContractConnector)))
|
|
mux.Handle("DELETE /api/secret-engines/{name}", guard(m.remove(store.ContractSecretEngine)))
|
|
mux.Handle("DELETE /api/mcp-servers/{name}", guard(m.remove(store.ContractMCP)))
|
|
mux.Handle("PUT /api/credentials/{name}", guard(http.HandlerFunc(m.putCredential)))
|
|
mux.Handle("GET /api/credentials", guard(http.HandlerFunc(m.listCredentials)))
|
|
}
|
|
|
|
type acceptRequest struct {
|
|
Path string `json:"path"`
|
|
SHA256 string `json:"sha256"`
|
|
Egress []string `json:"egress"`
|
|
Env []string `json:"env"`
|
|
Settings map[string]string `json:"settings"`
|
|
// Credentials maps each credential slot the package declares to a
|
|
// credential by name.
|
|
Credentials map[string]string `json:"credentials"`
|
|
}
|
|
|
|
func (m *Module) accept(contract string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) { m.acceptAs(w, r, contract) }
|
|
}
|
|
|
|
func (m *Module) acceptAs(w http.ResponseWriter, r *http.Request, contract string) {
|
|
name := r.PathValue("name")
|
|
var req acceptRequest
|
|
if err := httpjson.Read(r, &req); err != nil {
|
|
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
if !validName.MatchString(name) || req.Path == "" || req.SHA256 == "" {
|
|
httpjson.Error(w, http.StatusBadRequest, "a connector needs a lowercase name, a path and a sha256")
|
|
return
|
|
}
|
|
settings, err := json.Marshal(req.Settings)
|
|
if err != nil {
|
|
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
slots, err := json.Marshal(req.Credentials, json.FormatNilMapAsNull(false))
|
|
if err != nil {
|
|
httpjson.Error(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
a := store.Accepted{
|
|
OrgID: m.orgID, Name: name, Path: req.Path, SHA256: req.SHA256,
|
|
Egress: req.Egress, Env: req.Env, Settings: settings, Contract: contract, Credentials: slots,
|
|
// Only an MCP server declares its tools at acceptance; the column
|
|
// holds a list either way.
|
|
Tools: jsontext.Value(`[]`),
|
|
}
|
|
if err := m.store.Upsert(r.Context(), a); err != nil {
|
|
httpjson.Error(w, http.StatusInternalServerError, err.Error())
|
|
return
|
|
}
|
|
if err := m.launch(context.WithoutCancel(r.Context()), a); err != nil {
|
|
httpjson.Error(w, http.StatusUnprocessableEntity, err.Error())
|
|
return
|
|
}
|
|
httpjson.Write(w, http.StatusOK, map[string]string{"name": name, "state": "running"})
|
|
}
|
|
|
|
// launch starts or restarts one accepted package and records the outcome. A
|
|
// name runs one package: whatever ran under it before is stopped first, so a
|
|
// connector, a secret engine and an MCP server never share a name.
|
|
func (m *Module) launch(ctx context.Context, a store.Accepted) error {
|
|
m.release(a.Name)
|
|
if a.Contract == store.ContractMCP {
|
|
return m.startServer(ctx, a)
|
|
}
|
|
var settings map[string]string
|
|
if len(a.Settings) > 0 {
|
|
if err := json.Unmarshal(a.Settings, &settings); err != nil {
|
|
return m.fail(ctx, a.Name, fmt.Errorf("settings: %w", err))
|
|
}
|
|
}
|
|
secrets, err := m.secrets(ctx, a)
|
|
if err != nil {
|
|
return m.fail(ctx, a.Name, err)
|
|
}
|
|
spec := Spec{
|
|
Name: a.Name, Path: a.Path, SHA256: a.SHA256,
|
|
Env: a.Env, Egress: a.Egress, Settings: settings, Secrets: secrets,
|
|
}
|
|
if a.Contract == store.ContractSecretEngine {
|
|
e, err := m.host.StartEngine(ctx, spec)
|
|
if err != nil {
|
|
return m.fail(ctx, a.Name, err)
|
|
}
|
|
m.mu.Lock()
|
|
m.engines[a.Name] = e
|
|
delete(m.failed, a.Name)
|
|
m.mu.Unlock()
|
|
m.log.InfoContext(ctx, "secret engine started", "engine", a.Name)
|
|
return nil
|
|
}
|
|
c, err := m.host.Start(ctx, spec)
|
|
if err != nil {
|
|
return m.fail(ctx, a.Name, err)
|
|
}
|
|
m.mu.Lock()
|
|
m.running[a.Name] = c
|
|
delete(m.failed, a.Name)
|
|
m.mu.Unlock()
|
|
m.log.InfoContext(ctx, "connector started", "connector", a.Name)
|
|
return nil
|
|
}
|
|
|
|
// release stops whatever ran under name, so one accepted package is what a
|
|
// name runs.
|
|
func (m *Module) release(name string) {
|
|
m.mu.Lock()
|
|
_, running := m.running[name]
|
|
_, engine := m.engines[name]
|
|
s, stop := m.servers[name], m.stops[name]
|
|
delete(m.running, name)
|
|
delete(m.engines, name)
|
|
delete(m.servers, name)
|
|
delete(m.stops, name)
|
|
m.mu.Unlock()
|
|
if running || engine {
|
|
m.host.Stop(name)
|
|
}
|
|
if stop != nil {
|
|
stop()
|
|
}
|
|
if s != nil {
|
|
s.Close()
|
|
}
|
|
}
|
|
|
|
func (m *Module) fail(ctx context.Context, name string, err error) error {
|
|
m.mu.Lock()
|
|
delete(m.running, name)
|
|
delete(m.engines, name)
|
|
m.failed[name] = err.Error()
|
|
m.mu.Unlock()
|
|
m.log.ErrorContext(ctx, "extension failed to start", "extension", name, "error", err)
|
|
return err
|
|
}
|