feat(connectors): the connector contract and its confined extension host (#1979)

This commit is contained in:
sam committed 2026-11-15 17:24:31 +11:00
1 parent 8b64ab0afe
commit 58858f0d92
15 files changed
+3147 -1

No files matched your search

+8 -1
View File
@@ -113,7 +113,14 @@ linters:
- {pkg: "github.com/samcharles93/archie-core/internal/webui", desc: "old service code; move what you need into a module"}
- {pkg: "github.com/samcharles93/archie-core/internal/domain", desc: "old domain code; move what you need into a module"}
- {pkg: "github.com/samcharles93/archie-core/internal/infrastructure", desc: "old infrastructure; move what you need into a module"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts", desc: "internal service contracts are being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/state", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/gateway", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/daemon", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/controlplane", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/messaging", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/logging", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/forge", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/contracts/channel", desc: "an internal service contract being deleted"}
- {pkg: "github.com/samcharles93/archie-core/internal/config", desc: "the single binary has its own bootstrap config"}
- {pkg: "github.com/nats-io", desc: "the single binary has no message bus"}
shared:
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,343 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.6.2
// - protoc (unknown)
// source: connector/v1/connector.proto
package connectorv1
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.64.0 or later.
const _ = grpc.SupportPackageIsVersion9
const (
ConnectorService_Configure_FullMethodName = "/connector.v1.ConnectorService/Configure"
ConnectorService_Describe_FullMethodName = "/connector.v1.ConnectorService/Describe"
ConnectorService_Parse_FullMethodName = "/connector.v1.ConnectorService/Parse"
ConnectorService_Resolve_FullMethodName = "/connector.v1.ConnectorService/Resolve"
ConnectorService_Invoke_FullMethodName = "/connector.v1.ConnectorService/Invoke"
ConnectorService_Lookup_FullMethodName = "/connector.v1.ConnectorService/Lookup"
)
// ConnectorServiceClient is the client API for ConnectorService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// ConnectorService is the surface every connector extension serves: it
// declares what it hears and does, turns verified payloads into typed events,
// resolves and runs actions, and answers lookups. The host invokes everything
// by name; payloads are JSON validated against the declared schemas.
type ConnectorServiceClient interface {
// Configure hands the connector its operator settings and resolved secrets.
// The host calls it once after launch and again when settings change.
Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error)
// Describe declares the connector's event types, actions and lookups.
Describe(ctx context.Context, in *DescribeRequest, opts ...grpc.CallOption) (*DescribeResponse, error)
// Parse turns a payload the host has already verified into typed events.
Parse(ctx context.Context, in *ParseRequest, opts ...grpc.CallOption) (*ParseResponse, error)
// Resolve works out the structured target and the effect of one concrete
// action before it is judged. The same action can have different effects on
// different targets.
Resolve(ctx context.Context, in *ResolveRequest, opts ...grpc.CallOption) (*ResolveResponse, error)
// Invoke runs one judged action. The connector honours the idempotency key
// and returns the digest of what it actually executed.
Invoke(ctx context.Context, in *InvokeRequest, opts ...grpc.CallOption) (*InvokeResponse, error)
// Lookup answers a read-only question. Lookups change nothing and run no
// guards; a lookup that finds nothing says so.
Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error)
}
type connectorServiceClient struct {
cc grpc.ClientConnInterface
}
func NewConnectorServiceClient(cc grpc.ClientConnInterface) ConnectorServiceClient {
return &connectorServiceClient{cc}
}
func (c *connectorServiceClient) Configure(ctx context.Context, in *ConfigureRequest, opts ...grpc.CallOption) (*ConfigureResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ConfigureResponse)
err := c.cc.Invoke(ctx, ConnectorService_Configure_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *connectorServiceClient) Describe(ctx context.Context, in *DescribeRequest, opts ...grpc.CallOption) (*DescribeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(DescribeResponse)
err := c.cc.Invoke(ctx, ConnectorService_Describe_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *connectorServiceClient) Parse(ctx context.Context, in *ParseRequest, opts ...grpc.CallOption) (*ParseResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ParseResponse)
err := c.cc.Invoke(ctx, ConnectorService_Parse_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *connectorServiceClient) Resolve(ctx context.Context, in *ResolveRequest, opts ...grpc.CallOption) (*ResolveResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ResolveResponse)
err := c.cc.Invoke(ctx, ConnectorService_Resolve_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *connectorServiceClient) Invoke(ctx context.Context, in *InvokeRequest, opts ...grpc.CallOption) (*InvokeResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(InvokeResponse)
err := c.cc.Invoke(ctx, ConnectorService_Invoke_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *connectorServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(LookupResponse)
err := c.cc.Invoke(ctx, ConnectorService_Lookup_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// ConnectorServiceServer is the server API for ConnectorService service.
// All implementations must embed UnimplementedConnectorServiceServer
// for forward compatibility.
//
// ConnectorService is the surface every connector extension serves: it
// declares what it hears and does, turns verified payloads into typed events,
// resolves and runs actions, and answers lookups. The host invokes everything
// by name; payloads are JSON validated against the declared schemas.
type ConnectorServiceServer interface {
// Configure hands the connector its operator settings and resolved secrets.
// The host calls it once after launch and again when settings change.
Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error)
// Describe declares the connector's event types, actions and lookups.
Describe(context.Context, *DescribeRequest) (*DescribeResponse, error)
// Parse turns a payload the host has already verified into typed events.
Parse(context.Context, *ParseRequest) (*ParseResponse, error)
// Resolve works out the structured target and the effect of one concrete
// action before it is judged. The same action can have different effects on
// different targets.
Resolve(context.Context, *ResolveRequest) (*ResolveResponse, error)
// Invoke runs one judged action. The connector honours the idempotency key
// and returns the digest of what it actually executed.
Invoke(context.Context, *InvokeRequest) (*InvokeResponse, error)
// Lookup answers a read-only question. Lookups change nothing and run no
// guards; a lookup that finds nothing says so.
Lookup(context.Context, *LookupRequest) (*LookupResponse, error)
mustEmbedUnimplementedConnectorServiceServer()
}
// UnimplementedConnectorServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedConnectorServiceServer struct{}
func (UnimplementedConnectorServiceServer) Configure(context.Context, *ConfigureRequest) (*ConfigureResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Configure not implemented")
}
func (UnimplementedConnectorServiceServer) Describe(context.Context, *DescribeRequest) (*DescribeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Describe not implemented")
}
func (UnimplementedConnectorServiceServer) Parse(context.Context, *ParseRequest) (*ParseResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Parse not implemented")
}
func (UnimplementedConnectorServiceServer) Resolve(context.Context, *ResolveRequest) (*ResolveResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Resolve not implemented")
}
func (UnimplementedConnectorServiceServer) Invoke(context.Context, *InvokeRequest) (*InvokeResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Invoke not implemented")
}
func (UnimplementedConnectorServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) {
return nil, status.Error(codes.Unimplemented, "method Lookup not implemented")
}
func (UnimplementedConnectorServiceServer) mustEmbedUnimplementedConnectorServiceServer() {}
func (UnimplementedConnectorServiceServer) testEmbeddedByValue() {}
// UnsafeConnectorServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ConnectorServiceServer will
// result in compilation errors.
type UnsafeConnectorServiceServer interface {
mustEmbedUnimplementedConnectorServiceServer()
}
func RegisterConnectorServiceServer(s grpc.ServiceRegistrar, srv ConnectorServiceServer) {
// If the following call panics, it indicates UnimplementedConnectorServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&ConnectorService_ServiceDesc, srv)
}
func _ConnectorService_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConfigureRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Configure(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Configure_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Configure(ctx, req.(*ConfigureRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ConnectorService_Describe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DescribeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Describe(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Describe_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Describe(ctx, req.(*DescribeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ConnectorService_Parse_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ParseRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Parse(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Parse_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Parse(ctx, req.(*ParseRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ConnectorService_Resolve_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ResolveRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Resolve(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Resolve_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Resolve(ctx, req.(*ResolveRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ConnectorService_Invoke_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(InvokeRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Invoke(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Invoke_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Invoke(ctx, req.(*InvokeRequest))
}
return interceptor(ctx, in, info, handler)
}
func _ConnectorService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(LookupRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ConnectorServiceServer).Lookup(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ConnectorService_Lookup_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ConnectorServiceServer).Lookup(ctx, req.(*LookupRequest))
}
return interceptor(ctx, in, info, handler)
}
// ConnectorService_ServiceDesc is the grpc.ServiceDesc for ConnectorService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var ConnectorService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "connector.v1.ConnectorService",
HandlerType: (*ConnectorServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Configure",
Handler: _ConnectorService_Configure_Handler,
},
{
MethodName: "Describe",
Handler: _ConnectorService_Describe_Handler,
},
{
MethodName: "Parse",
Handler: _ConnectorService_Parse_Handler,
},
{
MethodName: "Resolve",
Handler: _ConnectorService_Resolve_Handler,
},
{
MethodName: "Invoke",
Handler: _ConnectorService_Invoke_Handler,
},
{
MethodName: "Lookup",
Handler: _ConnectorService_Lookup_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "connector/v1/connector.proto",
}
+296
View File
@@ -0,0 +1,296 @@
package connectors
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/google/jsonschema-go/jsonschema"
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
)
var (
// ErrDigestMismatch means the connector ran something other than the
// judged action. It is a finding and tightens the action class.
ErrDigestMismatch = errors.New("connector executed a different action than the one judged")
// ErrOutcomeUnknown means the connector can't confirm what its effect did.
ErrOutcomeUnknown = errors.New("connector can't confirm the outcome")
// ErrNotIdempotent refuses an action with external effects from a
// connector that doesn't honour idempotency keys.
ErrNotIdempotent = errors.New("action has external effects but the connector doesn't declare idempotency")
// ErrUnknownName names an action, lookup or event type the connector
// never declared.
ErrUnknownName = errors.New("not declared by the connector")
)
// Reversibility says whether and how an action can be undone.
type Reversibility int
// Reversibility values, in the order of decreasing autonomy they allow.
const (
ReversibilityUnspecified Reversibility = iota
UndoneAutomatically
Recoverable
Irreversible
)
// Effect describes what one action does.
type Effect struct {
ReadOnly bool
Reversibility Reversibility
Summary string
}
// Target is what an action acts on. Connector, action, kind and environment
// together are the action class.
type Target struct {
Kind string
ID string
Environment string
Disposable bool
}
// Resolution is an action's target and the effect it would have there.
type Resolution struct {
Target Target
Effect Effect
}
// Event is one typed event a connector parsed from a verified payload.
type Event struct {
Type string
SourceEventID string
Fields json.RawMessage
LinkKey string
OccurredAt time.Time
}
// Invocation is one judged action to run.
type Invocation struct {
Action string
Input json.RawMessage
IdempotencyKey string
}
type action struct {
effect Effect
idempotent bool
input *jsonschema.Resolved
}
// Connector is a running connector and what it declared.
type Connector struct {
name string
client pb.ConnectorServiceClient
events map[string]*jsonschema.Resolved
actions map[string]action
lookups map[string]*jsonschema.Resolved
}
// open configures a connector and reads its declaration.
func open(ctx context.Context, name string, client pb.ConnectorServiceClient, settings, secrets map[string]string) (*Connector, error) {
if _, err := client.Configure(ctx, &pb.ConfigureRequest{Settings: settings, Secrets: secrets}); err != nil {
return nil, fmt.Errorf("connector %q: configure: %w", name, err)
}
desc, err := client.Describe(ctx, &pb.DescribeRequest{})
if err != nil {
return nil, fmt.Errorf("connector %q: describe: %w", name, err)
}
if desc.GetContract() != contractVersion {
return nil, fmt.Errorf("connector %q speaks contract %q, the host speaks %q", name, desc.GetContract(), contractVersion)
}
c := &Connector{
name: name, client: client,
events: map[string]*jsonschema.Resolved{},
actions: map[string]action{},
lookups: map[string]*jsonschema.Resolved{},
}
return c, c.load(desc)
}
func (c *Connector) load(desc *pb.DescribeResponse) error {
for _, e := range desc.GetEvents() {
s, err := compile(e.GetSchema())
if err != nil {
return fmt.Errorf("connector %q: event %q: %w", c.name, e.GetName(), err)
}
c.events[e.GetName()] = s
}
for _, a := range desc.GetActions() {
s, err := compile(a.GetInputSchema())
if err != nil {
return fmt.Errorf("connector %q: action %q: %w", c.name, a.GetName(), err)
}
c.actions[a.GetName()] = action{effect: effectFrom(a.GetEffect()), idempotent: a.GetIdempotent(), input: s}
}
for _, l := range desc.GetLookups() {
s, err := compile(l.GetInputSchema())
if err != nil {
return fmt.Errorf("connector %q: lookup %q: %w", c.name, l.GetName(), err)
}
c.lookups[l.GetName()] = s
}
return nil
}
// Name is the connector's accepted name.
func (c *Connector) Name() string { return c.name }
// Parse turns a payload the caller has already verified into typed events,
// each checked against its declared schema.
func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]string) ([]Event, error) {
resp, err := c.client.Parse(ctx, &pb.ParseRequest{Body: body, Headers: headers})
if err != nil {
return nil, fmt.Errorf("connector %q: parse: %w", c.name, err)
}
events := make([]Event, 0, len(resp.GetEvents()))
for _, e := range resp.GetEvents() {
schema, ok := c.events[e.GetType()]
if !ok {
return nil, fmt.Errorf("connector %q: event type %q: %w", c.name, e.GetType(), ErrUnknownName)
}
if err := validate(schema, e.GetFields()); err != nil {
return nil, fmt.Errorf("connector %q: event %q: %w", c.name, e.GetType(), err)
}
events = append(events, Event{
Type: e.GetType(), SourceEventID: e.GetSourceEventId(), Fields: e.GetFields(),
LinkKey: e.GetLinkKey(), OccurredAt: e.GetOccurredAt().AsTime(),
})
}
return events, nil
}
// Resolve returns the target and effect of one concrete action.
func (c *Connector) Resolve(ctx context.Context, name string, input json.RawMessage) (Resolution, error) {
if _, err := c.action(name, input); err != nil {
return Resolution{}, err
}
resp, err := c.client.Resolve(ctx, &pb.ResolveRequest{Action: name, Input: input})
if err != nil {
return Resolution{}, fmt.Errorf("connector %q: resolve %q: %w", c.name, name, err)
}
t := resp.GetTarget()
return Resolution{
Target: Target{Kind: t.GetKind(), ID: t.GetId(), Environment: t.GetEnvironment(), Disposable: t.GetDisposable()},
Effect: effectFrom(resp.GetEffect()),
}, nil
}
// Invoke runs one judged action and decodes its output into T. The connector
// must report the digest of what it ran; a different digest is refused.
func (c *Connector) Invoke[T any](ctx context.Context, inv Invocation) (T, error) {
var zero T
a, err := c.action(inv.Action, inv.Input)
if err != nil {
return zero, err
}
if !a.effect.ReadOnly && !a.idempotent {
return zero, fmt.Errorf("connector %q: action %q: %w", c.name, inv.Action, ErrNotIdempotent)
}
digest := Digest(inv.Action, inv.Input)
resp, err := c.client.Invoke(ctx, &pb.InvokeRequest{
Action: inv.Action, Input: inv.Input, IdempotencyKey: inv.IdempotencyKey, Digest: digest,
})
if err != nil {
return zero, fmt.Errorf("connector %q: invoke %q: %w", c.name, inv.Action, err)
}
if resp.GetExecutedDigest() != digest {
return zero, fmt.Errorf("connector %q: action %q: %w", c.name, inv.Action, ErrDigestMismatch)
}
if resp.GetOutcomeUnknown() {
return zero, fmt.Errorf("connector %q: action %q: %w", c.name, inv.Action, ErrOutcomeUnknown)
}
return decode[T](resp.GetOutput())
}
// Lookup answers a read-only question and decodes the answer into T. found is
// false when the connector found nothing, which callers report as a finding.
func (c *Connector) Lookup[T any](ctx context.Context, name string, input json.RawMessage) (T, bool, error) {
var zero T
schema, ok := c.lookups[name]
if !ok {
return zero, false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(schema, input); err != nil {
return zero, false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
}
resp, err := c.client.Lookup(ctx, &pb.LookupRequest{Lookup: name, Input: input})
if err != nil {
return zero, false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
}
if !resp.GetFound() {
return zero, false, nil
}
out, err := decode[T](resp.GetOutput())
return out, err == nil, err
}
func (c *Connector) action(name string, input json.RawMessage) (action, error) {
a, ok := c.actions[name]
if !ok {
return action{}, fmt.Errorf("connector %q: action %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(a.input, input); err != nil {
return action{}, fmt.Errorf("connector %q: action %q: %w", c.name, name, err)
}
return a, nil
}
// Digest identifies one concrete action: its name and its exact input. The
// connector SDK computes the same digest over what it ran.
func Digest(action string, input json.RawMessage) string {
h := sha256.New()
h.Write([]byte(action))
h.Write([]byte{0})
h.Write(input)
return hex.EncodeToString(h.Sum(nil))
}
func effectFrom(e *pb.Effect) Effect {
return Effect{ReadOnly: e.GetReadOnly(), Reversibility: Reversibility(e.GetReversibility()), Summary: e.GetSummary()}
}
func compile(raw []byte) (*jsonschema.Resolved, error) {
if len(raw) == 0 {
raw = []byte(`{}`)
}
var s jsonschema.Schema
if err := json.Unmarshal(raw, &s); err != nil {
return nil, fmt.Errorf("schema: %w", err)
}
resolved, err := s.Resolve(nil)
if err != nil {
return nil, fmt.Errorf("schema: %w", err)
}
return resolved, nil
}
func validate(schema *jsonschema.Resolved, raw json.RawMessage) error {
if len(raw) == 0 {
raw = json.RawMessage(`{}`)
}
var instance any
if err := json.Unmarshal(raw, &instance); err != nil {
return fmt.Errorf("payload is not JSON: %w", err)
}
if err := schema.Validate(instance); err != nil {
return fmt.Errorf("payload doesn't match its schema: %w", err)
}
return nil
}
func decode[T any](raw []byte) (T, error) {
var out T
if len(raw) == 0 {
return out, nil
}
if err := json.Unmarshal(raw, &out); err != nil {
return out, fmt.Errorf("decode connector output: %w", err)
}
return out, nil
}
+96
View File
@@ -0,0 +1,96 @@
package connectors
import (
"context"
"encoding/json"
"errors"
"testing"
"google.golang.org/grpc"
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
)
// fakeClient is a connector that declares one action and answers Invoke as
// told, so the host's checks can be driven directly.
type fakeClient struct {
pb.ConnectorServiceClient
readOnly bool
idempotent bool
// executed overrides the digest the connector reports; empty echoes the
// host's digest.
executed string
unknown bool
invoked int
}
func (f *fakeClient) Configure(context.Context, *pb.ConfigureRequest, ...grpc.CallOption) (*pb.ConfigureResponse, error) {
return &pb.ConfigureResponse{}, nil
}
func (f *fakeClient) Describe(context.Context, *pb.DescribeRequest, ...grpc.CallOption) (*pb.DescribeResponse, error) {
return &pb.DescribeResponse{Contract: contractVersion, Actions: []*pb.ActionType{{
Name: "delete",
InputSchema: []byte(`{"type":"object","required":["path"],"properties":{"path":{"type":"string"}}}`),
Effect: &pb.Effect{ReadOnly: f.readOnly, Reversibility: pb.Reversibility_REVERSIBILITY_RECOVERABLE},
Idempotent: f.idempotent,
}}}, nil
}
func (f *fakeClient) Invoke(_ context.Context, req *pb.InvokeRequest, _ ...grpc.CallOption) (*pb.InvokeResponse, error) {
f.invoked++
executed := req.GetDigest()
if f.executed != "" {
executed = f.executed
}
return &pb.InvokeResponse{Output: []byte(`{"deleted":3}`), ExecutedDigest: executed, OutcomeUnknown: f.unknown}, nil
}
// TestInvokeRunsOnlyTheJudgedAction guards R18 and 2.5's checks at the
// connector boundary: input that breaks the declared schema never reaches the
// connector, an action with external effects is refused unless the connector
// honours idempotency keys, a connector that ran something other than the
// judged action is caught by its digest, and an unconfirmed outcome is
// reported as unknown rather than success.
func TestInvokeRunsOnlyTheJudgedAction(t *testing.T) {
good := json.RawMessage(`{"path":"/tmp/runner"}`)
tests := []struct {
name string
client *fakeClient
input json.RawMessage
wantErr error
wantAnyErr bool
wantInvoked int
}{
{name: "matching digest returns the output", client: &fakeClient{idempotent: true}, input: good, wantInvoked: 1},
{name: "a different executed digest is refused", client: &fakeClient{idempotent: true, executed: "other"}, input: good, wantErr: ErrDigestMismatch, wantInvoked: 1},
{name: "external effects without idempotency are refused before running", client: &fakeClient{}, input: good, wantErr: ErrNotIdempotent},
{name: "read-only actions need no idempotency", client: &fakeClient{readOnly: true}, input: good, wantInvoked: 1},
{name: "an unconfirmed outcome is unknown", client: &fakeClient{idempotent: true, unknown: true}, input: good, wantErr: ErrOutcomeUnknown, wantInvoked: 1},
{name: "input breaking the schema never reaches the connector", client: &fakeClient{idempotent: true}, input: json.RawMessage(`{"path":7}`), wantAnyErr: true, wantInvoked: 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c, err := open(t.Context(), "fake", tt.client, nil, nil)
if err != nil {
t.Fatalf("open: %v", err)
}
out, err := c.Invoke[map[string]int](t.Context(), Invocation{Action: "delete", Input: tt.input, IdempotencyKey: "run:1"})
wantOK := tt.wantErr == nil && !tt.wantAnyErr
switch {
case wantOK && err != nil:
t.Fatalf("Invoke: %v", err)
case !wantOK && err == nil:
t.Fatal("Invoke succeeded, want an error")
case tt.wantErr != nil && !errors.Is(err, tt.wantErr):
t.Fatalf("Invoke error = %v, want %v", err, tt.wantErr)
}
if wantOK && out["deleted"] != 3 {
t.Fatalf("output = %v, want deleted:3", out)
}
if tt.client.invoked != tt.wantInvoked {
t.Fatalf("connector invoked %d times, want %d", tt.client.invoked, tt.wantInvoked)
}
})
}
}
+93
View File
@@ -0,0 +1,93 @@
// Package connectors hosts connector extensions: one confined subprocess per
// accepted connector, speaking the connector contract (proto/connector/v1).
// Other modules reach a connector only through the Connector this package
// returns, which validates every payload against the connector's declared
// schemas and checks that only the judged action runs.
package connectors
import (
"context"
"fmt"
"log/slog"
goplugin "github.com/hashicorp/go-plugin"
"google.golang.org/grpc"
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
"github.com/samcharles93/archie-core/modules/connectors/internal/host"
)
// surface names the connector contract in the plugin handshake.
const surface = "connector"
// contractVersion is the connector contract this host speaks.
const contractVersion = "v1"
// Spec is one accepted connector to run.
type Spec struct {
Name string
// Path is the connector binary; SHA256 is the digest it must match.
Path string
SHA256 string
// Env names host environment variables the connector may see.
Env []string
// Egress is the accepted host list. A connector that touches targets
// declares it, and runs confined to those hosts.
Egress []string
Settings map[string]string
Secrets map[string]string
}
// Host owns the running connectors.
type Host struct {
procs *host.Host
}
// NewHost returns a host that logs plugin lifecycle to log.
func NewHost(log *slog.Logger) *Host {
return &Host{procs: host.NewHost(log)}
}
// Start launches spec, configures it and reads its declaration. A connector
// already running under the same name is replaced.
func (h *Host) Start(ctx context.Context, spec Spec) (*Connector, error) {
raw, err := h.procs.Start(ctx, host.Spec{
Name: spec.Name, Path: spec.Path, SHA256: spec.SHA256, Env: spec.Env, Egress: spec.Egress,
}, surface, &plugin{})
if err != nil {
return nil, err
}
client, ok := raw.(pb.ConnectorServiceClient)
if !ok {
h.procs.Stop(spec.Name)
return nil, fmt.Errorf("connector %q: plugin is not a connector", spec.Name)
}
c, err := open(ctx, spec.Name, client, spec.Settings, spec.Secrets)
if err != nil {
h.procs.Stop(spec.Name)
return nil, err
}
return c, nil
}
// Stop ends the named connector.
func (h *Host) Stop(name string) { h.procs.Stop(name) }
// Alive reports whether the named connector's process is running.
func (h *Host) Alive(name string) bool { return h.procs.Alive(name) }
// Close stops every connector.
func (h *Host) Close() { h.procs.Close() }
// plugin is the host side of the go-plugin handshake for the contract.
type plugin struct {
goplugin.NetRPCUnsupportedPlugin
}
func (*plugin) GRPCServer(*goplugin.GRPCBroker, *grpc.Server) error {
return fmt.Errorf("connectors are served by extensions, not by the host")
}
func (*plugin) GRPCClient(_ context.Context, _ *goplugin.GRPCBroker, conn *grpc.ClientConn) (any, error) {
return pb.NewConnectorServiceClient(conn), nil
}
@@ -0,0 +1,88 @@
package egress
import (
"net"
"strconv"
"strings"
)
// pattern is one allow-list entry: an exact host, a host on one port,
// a single-label wildcard, or everything.
type pattern struct {
everything bool
suffix string // ".example.com" for "*.example.com"
host string
port int // 0 means any port
}
// rules is a compiled allow list. An empty one allows nothing.
type rules struct {
allow []pattern
}
func compilePatterns(entries []string) []pattern {
out := make([]pattern, 0, len(entries))
for _, e := range entries {
out = append(out, compilePattern(e))
}
return out
}
func compilePattern(entry string) pattern {
entry = strings.TrimSpace(entry)
if entry == "*" || entry == "**" {
return pattern{everything: true}
}
var p pattern
if h, portText, err := net.SplitHostPort(entry); err == nil {
if port, err := strconv.Atoi(portText); err == nil {
entry, p.port = h, port
}
}
entry = normalizeHost(entry)
if rest, ok := strings.CutPrefix(entry, "*."); ok {
p.suffix = "." + rest
return p
}
p.host = entry
return p
}
func normalizeHost(host string) string {
return strings.TrimSuffix(strings.ToLower(host), ".")
}
func (p pattern) matches(host string, port int) bool {
if p.everything {
return true
}
if p.port != 0 && p.port != port {
return false
}
if p.suffix != "" {
label, ok := strings.CutSuffix(host, p.suffix)
return ok && label != "" && !strings.Contains(label, ".") && net.ParseIP(host) == nil
}
return host == p.host
}
func (r rules) allows(host string, port int) bool {
host = normalizeHost(host)
for _, p := range r.allow {
if p.matches(host, port) {
return true
}
}
return false
}
// namesHost reports whether an allow entry names host exactly, as a DNS name
// and not through a wildcard.
func (r rules) namesHost(host string) bool {
for _, p := range r.allow {
if !p.everything && p.suffix == "" && p.host == host {
return true
}
}
return false
}
@@ -0,0 +1,148 @@
// Package egress is the allow-list tunnel an extension's confined network
// namespace reaches its accepted hosts through.
package egress
import (
"context"
"fmt"
"io"
"net"
"net/http"
"net/http/httputil"
"strconv"
"sync"
)
// Tunnel is an allow-list forward proxy that never terminates TLS: CONNECT is
// spliced byte for byte to an admitted host, and absolute-form http requests
// are forwarded. Extensions get it rather than Proxy because they carry their
// own credentials and never trust the daemon CA.
type Tunnel struct {
rules rules
dial func(ctx context.Context, network, addr string) (net.Conn, error)
}
// NewTunnel admits the hosts allow names, in network-policy@1 form. A host
// named exactly may resolve to a private address, since the operator named
// it (a LAN forge); a wildcard match may not. Loopback is refused whatever
// is named: it would reach archie's own services.
func NewTunnel(allow []string) *Tunnel {
t := &Tunnel{rules: rules{allow: compilePatterns(allow)}}
t.dial = t.dialGuarded
return t
}
func (t *Tunnel) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defaultPort, hostport := 443, r.Host
if r.Method != http.MethodConnect {
if !r.URL.IsAbs() || r.URL.Scheme != "http" {
http.Error(w, "only CONNECT and absolute-form http requests are proxied", http.StatusBadRequest)
return
}
defaultPort, hostport = 80, r.URL.Host
}
host, port, err := splitTarget(hostport, defaultPort)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
target := net.JoinHostPort(host, strconv.Itoa(port))
if !t.rules.allows(host, port) {
deny(w, r, "egress to "+target+" is not allowed")
return
}
if r.Method == http.MethodConnect {
t.splice(w, r, target)
return
}
rp := &httputil.ReverseProxy{
Transport: &http.Transport{DialContext: t.dial},
FlushInterval: -1,
Rewrite: func(pr *httputil.ProxyRequest) {
pr.Out.URL.Host = target
pr.Out.Header.Del("Proxy-Connection")
},
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
},
}
rp.ServeHTTP(w, r)
}
func (t *Tunnel) splice(w http.ResponseWriter, r *http.Request, target string) {
upstream, err := t.dial(r.Context(), "tcp", target)
if err != nil {
http.Error(w, "upstream "+target+": "+err.Error(), http.StatusBadGateway)
return
}
hijacker, ok := w.(http.Hijacker)
if !ok {
_ = upstream.Close()
http.Error(w, "connection cannot be tunnelled", http.StatusInternalServerError)
return
}
client, buffered, err := hijacker.Hijack()
if err != nil {
_ = upstream.Close()
return
}
defer func() { _ = client.Close(); _ = upstream.Close() }()
if _, err := client.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
return
}
var wg sync.WaitGroup
wg.Go(func() {
_, _ = io.Copy(upstream, buffered)
closeWrite(upstream)
})
_, _ = io.Copy(client, upstream)
closeWrite(client)
wg.Wait()
}
// deny refuses a request. Go's client discards the body of a refused CONNECT
// and reports only the reason phrase, so for CONNECT the message is written
// as the reason phrase; otherwise a denial reads as a bare "Forbidden".
func deny(w http.ResponseWriter, r *http.Request, msg string) {
hijacker, ok := w.(http.Hijacker)
if r.Method != http.MethodConnect || !ok {
http.Error(w, msg, http.StatusForbidden)
return
}
conn, _, err := hijacker.Hijack()
if err != nil {
return
}
defer func() { _ = conn.Close() }()
_, _ = fmt.Fprintf(conn, "HTTP/1.1 403 %s\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", msg)
}
func closeWrite(c net.Conn) {
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
return
}
_ = c.Close()
}
// dialGuarded applies the Proxy's upstream guard, counting a host the allow
// list names exactly as naming every address it resolves to.
func (t *Tunnel) dialGuarded(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
named := t.rules.namesHost(normalizeHost(host))
addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
guardErr := fmt.Errorf("%s resolved to no address", host)
for _, a := range addrs {
if guardErr = guardUpstream(a.IP, named); guardErr != nil {
continue
}
return (&net.Dialer{Timeout: handshakeTimeout}).DialContext(ctx, network, net.JoinHostPort(a.IP.String(), port))
}
return nil, guardErr
}
@@ -0,0 +1,54 @@
package egress
import (
"fmt"
"net"
"strconv"
"strings"
"time"
)
const handshakeTimeout = 10 * time.Second
// cgnat is shared address space (RFC 6598), which overlay networks use for
// their addresses; it is internal like the private ranges.
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)}
// guardUpstream refuses loopback and unspecified addresses always, and
// private, link-local, shared and multicast addresses unless the policy
// named that IP or the host it came from.
func guardUpstream(ip net.IP, trusted bool) error {
if ip.IsLoopback() || ip.IsUnspecified() {
return fmt.Errorf("upstream address %s is the host itself", ip)
}
internal := ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || cgnat.Contains(ip)
if internal && !trusted {
return fmt.Errorf("upstream address %s is internal and not named by the network policy", ip)
}
return nil
}
func splitTarget(hostport string, defaultPort int) (string, int, error) {
if !hasPort(hostport) {
return normalizeHost(strings.Trim(hostport, "[]")), defaultPort, nil
}
host, portText, err := net.SplitHostPort(hostport)
if err != nil {
return "", 0, err
}
port, err := strconv.Atoi(portText)
if err != nil || port <= 0 || port > 65535 {
return "", 0, fmt.Errorf("invalid port in %q", hostport)
}
return normalizeHost(host), port, nil
}
// hasPort reports whether hostport ends in a port: a bracketed IPv6 literal
// followed by one, or a host with exactly one colon.
func hasPort(hostport string) bool {
if strings.HasPrefix(hostport, "[") {
return strings.Contains(hostport, "]:")
}
return strings.Count(hostport, ":") == 1
}
@@ -0,0 +1,168 @@
package host
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"os/exec"
"os/signal"
"runtime"
"syscall"
"golang.org/x/sys/unix"
)
// launcherArg makes any binary importing this package the confined
// launcher: the extension's parent inside its network namespace.
const launcherArg = "__extension-egress"
// proxyAddr is where the launcher serves the extension's only way out.
const proxyAddr = "127.0.0.1:3128"
func init() {
if len(os.Args) > 2 && os.Args[1] == launcherArg {
os.Exit(launch(os.Args[2]))
}
}
// confinedCommand re-execs this binary as the launcher in a fresh user and
// network namespace. The namespace holds only loopback, so the extension
// cannot reach anything except through the launcher's relay to socket. Where
// the kernel refuses the namespaces, the start fails: there is no unconfined
// fallback.
func confinedCommand(ctx context.Context, binary *os.File, socket string) (*exec.Cmd, error) {
self, err := os.Executable()
if err != nil {
return nil, err
}
uid, gid := os.Getuid(), os.Getgid()
cmd := exec.CommandContext(ctx, self, launcherArg, socket)
cmd.ExtraFiles = []*os.File{binary}
cmd.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWUSER | syscall.CLONE_NEWNET,
UidMappings: []syscall.SysProcIDMap{{ContainerID: uid, HostID: uid, Size: 1}},
GidMappings: []syscall.SysProcIDMap{{ContainerID: gid, HostID: gid, Size: 1}},
// Only the launcher holds this, to bring loopback up; it clears it
// before the extension starts.
AmbientCaps: []uintptr{unix.CAP_NET_ADMIN},
Pdeathsig: syscall.SIGKILL,
}
return cmd, nil
}
// openCommand runs the verified binary by its open descriptor with the host's
// network, for a package that declares no egress hosts.
func openCommand(ctx context.Context, binary *os.File) *exec.Cmd {
path := fmt.Sprintf("/proc/%d/fd/%d", os.Getpid(), binary.Fd())
cmd := exec.CommandContext(ctx, path) // the host-verified binary, by descriptor
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
return cmd
}
// launch runs inside the namespace: loopback up, relay listening, then the
// extension from descriptor 3 with every capability gone.
func launch(socket string) int {
ctx := context.Background()
if err := loopbackUp(); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: loopback:", err)
return 1
}
ln, err := (&net.ListenConfig{}).Listen(ctx, "tcp", proxyAddr)
if err != nil {
fmt.Fprintln(os.Stderr, "extension egress: relay:", err)
return 1
}
go relay(ctx, ln, socket)
if err := unix.Prctl(unix.PR_CAP_AMBIENT, unix.PR_CAP_AMBIENT_CLEAR_ALL, 0, 0, 0); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: drop capabilities:", err)
return 1
}
if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: no_new_privs:", err)
return 1
}
syscall.CloseOnExec(3)
return runExtension(ctx, fmt.Sprintf("/proc/%d/fd/3", os.Getpid()))
}
// runExtension starts the extension as the launcher's child and mirrors its
// exit. Pdeathsig ties the child to this thread, so killing the launcher
// kills the extension.
func runExtension(ctx context.Context, path string) int {
runtime.LockOSThread()
proxy := "http://" + proxyAddr
cmd := exec.CommandContext(ctx, path) // the host-verified binary, by descriptor
cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr
cmd.Env = append(os.Environ(),
"HTTP_PROXY="+proxy, "HTTPS_PROXY="+proxy, "ALL_PROXY="+proxy,
"http_proxy="+proxy, "https_proxy="+proxy, "all_proxy="+proxy, "NO_PROXY=", "no_proxy=")
cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL}
if err := cmd.Start(); err != nil {
fmt.Fprintln(os.Stderr, "extension egress: start:", err)
return 1
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
go func() {
for s := range signals {
_ = cmd.Process.Signal(s)
}
}()
err := cmd.Wait()
if exit, ok := errors.AsType[*exec.ExitError](err); ok {
return exit.ExitCode()
}
if err != nil {
return 1
}
return 0
}
func loopbackUp() error {
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return err
}
defer func() { _ = unix.Close(fd) }()
ifr, err := unix.NewIfreq("lo")
if err != nil {
return err
}
if err := unix.IoctlIfreq(fd, unix.SIOCGIFFLAGS, ifr); err != nil {
return err
}
ifr.SetUint16(ifr.Uint16() | unix.IFF_UP)
return unix.IoctlIfreq(fd, unix.SIOCSIFFLAGS, ifr)
}
// relay carries each proxy connection to the host's gate socket.
func relay(ctx context.Context, ln net.Listener, socket string) {
for {
conn, err := ln.Accept()
if err != nil {
return
}
go func() {
defer func() { _ = conn.Close() }()
upstream, err := (&net.Dialer{}).DialContext(ctx, "unix", socket)
if err != nil {
return
}
defer func() { _ = upstream.Close() }()
go func() {
_, _ = io.Copy(upstream, conn)
closeWrite(upstream)
}()
_, _ = io.Copy(conn, upstream)
}()
}
}
func closeWrite(c net.Conn) {
if cw, ok := c.(interface{ CloseWrite() error }); ok {
_ = cw.CloseWrite()
}
}
@@ -0,0 +1,20 @@
//go:build !linux
package host
import (
"context"
"errors"
"os"
"os/exec"
)
// confinedCommand refuses: extensions run only where their egress can be
// confined to the hosts the operator accepted.
func confinedCommand(context.Context, *os.File, string) (*exec.Cmd, error) {
return nil, errors.New("extensions need Linux network namespaces to confine their egress")
}
func openCommand(ctx context.Context, binary *os.File) *exec.Cmd {
return exec.CommandContext(ctx, binary.Name()) // the host-verified binary
}
+48
View File
@@ -0,0 +1,48 @@
package host
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/samcharles93/archie-core/modules/connectors/internal/egress"
)
// gate is the host end of one extension's egress: a Unix socket serving an
// allow-list proxy. A path socket is reachable from the extension's network
// namespace, where nothing else outside it is.
type gate struct {
dir string
socket string
server *http.Server
}
func openGate(ctx context.Context, allow []string) (*gate, error) {
dir, err := os.MkdirTemp("", "archie-ext-")
if err != nil {
return nil, fmt.Errorf("create egress directory in %s: %w", os.TempDir(), err)
}
g := &gate{dir: dir, socket: filepath.Join(dir, "egress.sock")}
ln, err := (&net.ListenConfig{}).Listen(ctx, "unix", g.socket)
if err != nil {
_ = os.RemoveAll(dir)
return nil, err
}
g.server = &http.Server{Handler: egress.NewTunnel(allow), ReadHeaderTimeout: 30 * time.Second}
go func() {
if err := g.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
_ = ln.Close()
}
}()
return g, nil
}
func (g *gate) close() {
_ = g.server.Close()
_ = os.RemoveAll(g.dir)
}
+119
View File
@@ -0,0 +1,119 @@
package host
import (
"context"
"io"
"log"
"log/slog"
"strings"
"github.com/hashicorp/go-hclog"
)
// slogger adapts a slog.Logger to the hclog.Logger go-plugin requires, so
// extension lifecycle and plugin output land in the service's own log at its
// configured level.
type slogger struct {
log *slog.Logger
name string
args []any
}
var _ hclog.Logger = slogger{}
func newSlogger(log *slog.Logger) hclog.Logger { return slogger{log: log} }
// levels maps hclog levels to slog; hclog's Trace sits below slog's Debug.
func level(l hclog.Level) slog.Level {
switch l {
case hclog.Trace:
return slog.LevelDebug - 4
case hclog.Debug:
return slog.LevelDebug
case hclog.Warn:
return slog.LevelWarn
case hclog.Error:
return slog.LevelError
default:
return slog.LevelInfo
}
}
func (s slogger) Log(l hclog.Level, msg string, args ...any) {
if s.name != "" {
args = append(args, "extension", s.name)
}
s.log.Log(context.Background(), level(l), msg, args...)
}
func (s slogger) Trace(msg string, args ...any) { s.Log(hclog.Trace, msg, args...) }
func (s slogger) Debug(msg string, args ...any) { s.Log(hclog.Debug, msg, args...) }
func (s slogger) Info(msg string, args ...any) { s.Log(hclog.Info, msg, args...) }
func (s slogger) Warn(msg string, args ...any) { s.Log(hclog.Warn, msg, args...) }
func (s slogger) Error(msg string, args ...any) { s.Log(hclog.Error, msg, args...) }
func (s slogger) enabled(l hclog.Level) bool {
return s.log.Enabled(context.Background(), level(l))
}
func (s slogger) IsTrace() bool { return s.enabled(hclog.Trace) }
func (s slogger) IsDebug() bool { return s.enabled(hclog.Debug) }
func (s slogger) IsInfo() bool { return s.enabled(hclog.Info) }
func (s slogger) IsWarn() bool { return s.enabled(hclog.Warn) }
func (s slogger) IsError() bool { return s.enabled(hclog.Error) }
func (s slogger) ImpliedArgs() []any { return s.args }
func (s slogger) With(args ...any) hclog.Logger {
s.log = s.log.With(args...)
s.args = append(append([]any(nil), s.args...), args...)
return s
}
func (s slogger) Name() string { return s.name }
func (s slogger) Named(name string) hclog.Logger {
if s.name != "" {
name = s.name + "." + name
}
return s.ResetNamed(name)
}
func (s slogger) ResetNamed(name string) hclog.Logger {
s.name = name
return s
}
// SetLevel is a no-op: the service's [log] level decides.
func (slogger) SetLevel(hclog.Level) {}
func (s slogger) GetLevel() hclog.Level {
for _, l := range []hclog.Level{hclog.Trace, hclog.Debug, hclog.Info, hclog.Warn} {
if s.enabled(l) {
return l
}
}
return hclog.Error
}
func (s slogger) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger {
return log.New(s.StandardWriter(opts), "", 0)
}
func (s slogger) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer {
l := hclog.Info
if opts != nil && opts.ForceLevel != hclog.NoLevel {
l = opts.ForceLevel
}
return lineWriter{s: s, level: l}
}
type lineWriter struct {
s slogger
level hclog.Level
}
func (w lineWriter) Write(p []byte) (int, error) {
w.s.Log(w.level, strings.TrimRight(string(p), "\n"))
return len(p), nil
}
+198
View File
@@ -0,0 +1,198 @@
// Package host supervises extension plugins: one go-plugin subprocess per
// enabled extension, each serving a gRPC surface from proto/<surface>/v1.
package host
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"os"
"os/exec"
"sync"
"github.com/hashicorp/go-hclog"
goplugin "github.com/hashicorp/go-plugin"
)
// Handshake is shared by the host and every extension binary. Bump
// ProtocolVersion on an incompatible change to the handshake itself; each
// surface versions its own proto package.
var Handshake = goplugin.HandshakeConfig{
ProtocolVersion: 1,
MagicCookieKey: "ARCHIE_EXTENSION",
MagicCookieValue: "archie",
}
// Spec is one extension to run.
type Spec struct {
Name string
// Path is the extension binary.
Path string
// SHA256 is the hex digest the binary must match before it is launched.
SHA256 string
// Env names the host environment variables the extension may see. PATH and
// HOME are always passed so a plugin can find the CLI it wraps.
Env []string
// Egress is the accepted egress host list. When it names hosts, the
// extension runs in its own network namespace and reaches only these,
// through a proxy; an empty list runs it on the host's network.
Egress []string
}
// Host owns the running extension processes, keyed by Spec.Name.
type Host struct {
log hclog.Logger
mu sync.Mutex
procs map[string]*proc
}
// proc is one running extension and the egress gate it alone can reach.
type proc struct {
client *goplugin.Client
gate *gate
}
func (p *proc) kill() {
p.client.Kill()
if p.gate != nil {
p.gate.close()
}
}
// command confines the extension to egress when the package declares hosts.
// A package that declares none opted out of confinement and gets the host's
// network, with no gate.
func command(ctx context.Context, binary *os.File, egress []string) (*exec.Cmd, *gate, error) {
if len(egress) == 0 {
return openCommand(ctx, binary), nil, nil
}
g, err := openGate(ctx, egress)
if err != nil {
return nil, nil, fmt.Errorf("egress: %w", err)
}
cmd, err := confinedCommand(ctx, binary, g.socket)
if err != nil {
g.close()
return nil, nil, err
}
return cmd, g, nil
}
// NewHost returns a Host that reports plugin lifecycle problems to log.
func NewHost(log *slog.Logger) *Host {
return &Host{log: newSlogger(log), procs: make(map[string]*proc)}
}
// Start launches spec serving surface and returns the dispensed client. An
// extension already running under the same name is stopped first, so Start is
// also how a changed binary or setting takes effect.
func (h *Host) Start(ctx context.Context, spec Spec, surface string, impl goplugin.Plugin) (any, error) {
sum, err := hex.DecodeString(spec.SHA256)
if err != nil || len(sum) != sha256.Size {
return nil, fmt.Errorf("extension %q: sha256 must be a hex digest", spec.Name)
}
// The verified file is what runs: the confined launcher execs this open
// descriptor, so swapping the path after the check changes nothing.
binary, err := openVerified(spec.Path, sum)
if err != nil {
return nil, fmt.Errorf("extension %q: %w", spec.Name, err)
}
defer func() { _ = binary.Close() }()
cmd, g, err := command(ctx, binary, spec.Egress)
if err != nil {
return nil, fmt.Errorf("extension %q: %w", spec.Name, err)
}
cmd.Env = passEnv(spec.Env)
p := &proc{gate: g, client: goplugin.NewClient(&goplugin.ClientConfig{
HandshakeConfig: Handshake,
Plugins: goplugin.PluginSet{surface: impl},
Cmd: cmd,
AllowedProtocols: []goplugin.Protocol{goplugin.ProtocolGRPC},
SkipHostEnv: true,
AutoMTLS: true,
Logger: h.log.Named(spec.Name),
})}
rpc, err := p.client.Client()
if err != nil {
p.kill()
return nil, fmt.Errorf("extension %q: connect: %w", spec.Name, err)
}
raw, err := rpc.Dispense(surface)
if err != nil {
p.kill()
return nil, fmt.Errorf("extension %q: dispense %s: %w", spec.Name, surface, err)
}
h.mu.Lock()
old := h.procs[spec.Name]
h.procs[spec.Name] = p
h.mu.Unlock()
if old != nil {
old.kill()
}
return raw, nil
}
// Stop kills the named extension. Stopping one that is not running is not an
// error.
func (h *Host) Stop(name string) {
h.mu.Lock()
p := h.procs[name]
delete(h.procs, name)
h.mu.Unlock()
if p != nil {
p.kill()
}
}
// Alive reports whether the named extension's process is still running.
func (h *Host) Alive(name string) bool {
h.mu.Lock()
defer h.mu.Unlock()
p := h.procs[name]
return p != nil && !p.client.Exited()
}
// Close stops every extension.
func (h *Host) Close() {
h.mu.Lock()
procs := h.procs
h.procs = make(map[string]*proc)
h.mu.Unlock()
for _, p := range procs {
p.kill()
}
}
// openVerified opens the extension binary and checks it against sum.
func openVerified(path string, sum []byte) (*os.File, error) {
f, err := os.Open(path) // the path is the installed package's binary
if err != nil {
return nil, err
}
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
_ = f.Close()
return nil, err
}
if !bytes.Equal(h.Sum(nil), sum) {
_ = f.Close()
return nil, errors.New("binary does not match its checksum")
}
return f, nil
}
func passEnv(names []string) []string {
env := make([]string, 0, len(names)+2)
for _, name := range append([]string{"PATH", "HOME"}, names...) {
if value, ok := os.LookupEnv(name); ok {
env = append(env, name+"="+value)
}
}
return env
}
+151
View File
@@ -0,0 +1,151 @@
syntax = "proto3";
package connector.v1;
import "google/protobuf/timestamp.proto";
// ConnectorService is the surface every connector extension serves: it
// declares what it hears and does, turns verified payloads into typed events,
// resolves and runs actions, and answers lookups. The host invokes everything
// by name; payloads are JSON validated against the declared schemas.
service ConnectorService {
// Configure hands the connector its operator settings and resolved secrets.
// The host calls it once after launch and again when settings change.
rpc Configure(ConfigureRequest) returns (ConfigureResponse);
// Describe declares the connector's event types, actions and lookups.
rpc Describe(DescribeRequest) returns (DescribeResponse);
// Parse turns a payload the host has already verified into typed events.
rpc Parse(ParseRequest) returns (ParseResponse);
// Resolve works out the structured target and the effect of one concrete
// action before it is judged. The same action can have different effects on
// different targets.
rpc Resolve(ResolveRequest) returns (ResolveResponse);
// Invoke runs one judged action. The connector honours the idempotency key
// and returns the digest of what it actually executed.
rpc Invoke(InvokeRequest) returns (InvokeResponse);
// Lookup answers a read-only question. Lookups change nothing and run no
// guards; a lookup that finds nothing says so.
rpc Lookup(LookupRequest) returns (LookupResponse);
}
message ConfigureRequest {
map<string, string> settings = 1;
map<string, string> secrets = 2;
}
message ConfigureResponse {}
message DescribeRequest {}
message DescribeResponse {
// contract is the contract version the connector speaks, "v1".
string contract = 1;
repeated EventType events = 2;
repeated ActionType actions = 3;
repeated LookupType lookups = 4;
}
message EventType {
string name = 1;
string description = 2;
// schema is the JSON Schema of the event's fields.
bytes schema = 3;
}
message ActionType {
string name = 1;
string description = 2;
// input_schema is the JSON Schema of the action's input.
bytes input_schema = 3;
// effect is the static declaration; Resolve refines it per action.
Effect effect = 4;
// idempotent declares that the connector honours idempotency keys. A
// connector that declares external effects without it has its actions
// refused.
bool idempotent = 5;
}
message LookupType {
string name = 1;
string description = 2;
bytes input_schema = 3;
}
enum Reversibility {
REVERSIBILITY_UNSPECIFIED = 0;
// REVERSIBILITY_UNDONE_AUTOMATICALLY can be reversed without help.
REVERSIBILITY_UNDONE_AUTOMATICALLY = 1;
// REVERSIBILITY_RECOVERABLE can be recovered from a snapshot, or touches
// only a disposable target.
REVERSIBILITY_RECOVERABLE = 2;
REVERSIBILITY_IRREVERSIBLE = 3;
}
message Effect {
bool read_only = 1;
Reversibility reversibility = 2;
// summary says in plain words what the action does, for example "delete
// files older than 24h under /tmp/runner".
string summary = 3;
}
// Target is what an action acts on. Connector, action, kind and environment
// together are the action class.
message Target {
string kind = 1;
string id = 2;
string environment = 3;
bool disposable = 4;
}
message ParseRequest {
bytes body = 1;
map<string, string> headers = 2;
}
message ParseResponse {
repeated Event events = 1;
}
message Event {
string type = 1;
// source_event_id is unique per source; a re-delivery repeats it.
string source_event_id = 2;
// fields is the event's JSON, matching its type's schema.
bytes fields = 3;
// link_key links this event to an earlier one it resolves or follows, such
// as Alertmanager's fingerprint.
string link_key = 4;
google.protobuf.Timestamp occurred_at = 5;
}
message ResolveRequest {
string action = 1;
bytes input = 2;
}
message ResolveResponse {
Target target = 1;
Effect effect = 2;
}
message InvokeRequest {
string action = 1;
bytes input = 2;
string idempotency_key = 3;
// digest is the host's digest of the judged action.
string digest = 4;
}
message InvokeResponse {
bytes output = 1;
// executed_digest is the connector's digest of what it ran. A mismatch is a
// finding and tightens the action class.
string executed_digest = 2;
// outcome_unknown says the connector can't confirm what its effect did.
bool outcome_unknown = 3;
}
message LookupRequest {
string lookup = 1;
bytes input = 2;
}
message LookupResponse {
bytes output = 1;
bool found = 2;
}