Files
archie/modules/connectors/connector.go
T

521 lines
18 KiB
Go

package connectors
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json/jsontext"
json "encoding/json/v2"
"errors"
"fmt"
"maps"
"slices"
"time"
"github.com/google/jsonschema-go/jsonschema"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
pb "github.com/samcharles93/archie-core/internal/contracts/connector/v1"
)
var (
// ErrDigestMismatch means the connector ran something other than the
// judged action. It is a finding and tightens the action class.
ErrDigestMismatch = errors.New("connector executed a different action than the one judged")
// ErrOutcomeUnknown means the connector can't confirm what its effect did.
ErrOutcomeUnknown = errors.New("connector can't confirm the outcome")
// ErrNotIdempotent refuses an action with external effects from a
// connector that doesn't honour idempotency keys.
ErrNotIdempotent = errors.New("action has external effects but the connector doesn't declare idempotency")
// ErrUnknownName names an action, lookup or event type the connector
// never declared.
ErrUnknownName = errors.New("not declared by the connector")
// ErrRejected means a delivery was rejected permanently, so retrying it
// cannot succeed: the connector returned InvalidArgument,
// FailedPrecondition, PermissionDenied or NotFound, or the input did not
// match the declared schema.
ErrRejected = errors.New("delivery permanently rejected")
)
// Reversibility says whether and how an action can be undone.
type Reversibility int
// Reversibility values, in the order of decreasing autonomy they allow.
const (
ReversibilityUnspecified Reversibility = iota
UndoneAutomatically
Recoverable
Irreversible
)
// Effect describes what one action does.
type Effect struct {
ReadOnly bool
Reversibility Reversibility
Summary string
}
// Target is what an action acts on. Connector, action, kind and environment
// together are the action class.
type Target struct {
Kind string
ID string
Environment string
Disposable bool
}
// Resolution is an action's target and the effect it would have there.
type Resolution struct {
Target Target
Effect Effect
}
// Event is one typed event a connector parsed from a verified payload.
type Event struct {
Type string
SourceEventID string
Fields jsontext.Value
LinkKey string
OccurredAt time.Time
}
// Invocation is one judged action to run.
type Invocation struct {
Action string
Input jsontext.Value
IdempotencyKey string
}
type action struct {
effect Effect
idempotent bool
input *jsonschema.Resolved
description string
schema jsontext.Value
}
type lookup struct {
input *jsonschema.Resolved
description string
schema jsontext.Value
}
// Declaration is one lookup or action a connector declared, as an agent is
// offered it.
type Declaration struct {
Connector string `json:"connector"`
Kind string `json:"kind"`
Name string `json:"name"`
Description string `json:"description"`
Schema jsontext.Value `json:"schema"`
ReadOnly bool `json:"read_only"`
// TrustLevel says whether the tool's output is trusted internal content.
// Anything but TrustTrusted, including empty, is treated as untrusted.
TrustLevel string `json:"trust_level,omitempty"`
}
// Trust levels a declaration can carry.
const (
TrustTrusted = "trusted"
TrustUntrusted = "untrusted"
)
// Declaration kinds.
const (
KindLookup = "lookup"
KindAction = "action"
)
// Tool runs one declaration by name: a connector's lookup or action, or a tool
// an accepted MCP server declares. Output is the text the agent is given,
// verbatim.
type Tool interface {
// Lookup answers a read-only question. found is false when there was
// nothing to find, which callers report as a finding.
Lookup(ctx context.Context, name string, input jsontext.Value) (output string, found bool, err error)
// Invoke runs one judged action.
Invoke(ctx context.Context, inv Invocation) (output string, err error)
// Resolve works out one concrete action's target and effect, for
// judgement before it runs.
Resolve(ctx context.Context, name string, input jsontext.Value) (Resolution, error)
}
// Connector is a running connector and what it declared.
type Connector struct {
name string
version string
client pb.ConnectorServiceClient
eventTypes map[string]*jsonschema.Resolved
actions map[string]action
lookups map[string]lookup
// deliveries are the messages and records it sends, never offered to an
// agent.
deliveries map[string]*jsonschema.Resolved
listens bool
// facts says it answers TargetFacts.
facts bool
// settings is the schema it declared for its operator settings.
settings []Setting
}
// open configures a connector and reads its declaration.
func open(ctx context.Context, name string, client pb.ConnectorServiceClient, settings, secrets map[string]string) (*Connector, error) {
if _, err := client.Configure(ctx, &pb.ConfigureRequest{Settings: settings, Secrets: secrets}); err != nil {
return nil, fmt.Errorf("connector %q: configure: %w", name, err)
}
desc, err := client.Describe(ctx, &pb.DescribeRequest{})
if err != nil {
return nil, fmt.Errorf("connector %q: describe: %w", name, err)
}
if desc.GetContract() != contractVersion {
return nil, fmt.Errorf("connector %q speaks contract %q, the host speaks %q", name, desc.GetContract(), contractVersion)
}
c := &Connector{
name: name, version: desc.GetVersion(), client: client,
eventTypes: map[string]*jsonschema.Resolved{},
actions: map[string]action{},
lookups: map[string]lookup{},
deliveries: map[string]*jsonschema.Resolved{},
listens: desc.GetListens(),
facts: desc.GetTargetFacts(),
settings: connectorSettings(desc.GetSettings()),
}
if err := validateSettingsSchema(c.settings); err != nil {
return nil, fmt.Errorf("connector %q: %w", name, err)
}
return c, c.load(desc)
}
func (c *Connector) load(desc *pb.DescribeResponse) error {
for _, e := range desc.GetEvents() {
s, err := compile(e.GetSchema())
if err != nil {
return fmt.Errorf("connector %q: event %q: %w", c.name, e.GetName(), err)
}
c.eventTypes[e.GetName()] = s
}
for _, a := range desc.GetActions() {
s, err := compile(a.GetInputSchema())
if err != nil {
return fmt.Errorf("connector %q: action %q: %w", c.name, a.GetName(), err)
}
c.actions[a.GetName()] = action{
effect: effectFrom(a.GetEffect()), idempotent: a.GetIdempotent(), input: s,
description: a.GetDescription(), schema: schemaOrEmpty(a.GetInputSchema()),
}
}
for _, l := range desc.GetLookups() {
s, err := compile(l.GetInputSchema())
if err != nil {
return fmt.Errorf("connector %q: lookup %q: %w", c.name, l.GetName(), err)
}
c.lookups[l.GetName()] = lookup{input: s, description: l.GetDescription(), schema: schemaOrEmpty(l.GetInputSchema())}
}
for _, d := range desc.GetDeliveries() {
s, err := compile(d.GetInputSchema())
if err != nil {
return fmt.Errorf("connector %q: delivery %q: %w", c.name, d.GetName(), err)
}
c.deliveries[d.GetName()] = s
}
return nil
}
// Delivers reports whether the connector sends the named delivery.
func (c *Connector) Delivers(name string) bool {
_, ok := c.deliveries[name]
return ok
}
// Deliver sends one delivery, checked against its declared schema, and
// returns what the service called it.
func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Value, key string) (string, error) {
schema, ok := c.deliveries[name]
if !ok {
return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(schema, input); err != nil {
return "", fmt.Errorf("connector %q: delivery %q: %w: %w", c.name, name, ErrRejected, err)
}
req := &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}
resp, err := c.client.Deliver(ctx, req)
recordProto(ctx, c.name, "deliver", req, resp, err)
if err != nil {
if rejectedForGood(err) {
return "", fmt.Errorf("connector %q: deliver %q: %w: %w", c.name, name, ErrRejected, err)
}
return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err)
}
return resp.GetExternalId(), nil
}
// rejectedForGood reports whether a connector's error status means that
// retrying the same delivery cannot succeed.
func rejectedForGood(err error) bool {
switch status.Code(err) {
case codes.InvalidArgument, codes.FailedPrecondition, codes.PermissionDenied, codes.NotFound:
return true
default:
return false
}
}
// Listens reports whether the connector holds its own connection to a source
// and streams its events.
func (c *Connector) Listens() bool { return c.listens }
// Listen streams the connector's events from cursor on, handing each batch
// and the cursor after it to store. It returns when ctx ends, the stream
// breaks, or store fails; the caller listens again from its last cursor.
func (c *Connector) Listen(ctx context.Context, cursor string, store func(events []Event, cursor string) error) error {
stream, err := c.client.Listen(ctx, &pb.ListenRequest{Cursor: cursor})
if err != nil {
return fmt.Errorf("connector %q: listen: %w", c.name, err)
}
for {
resp, err := stream.Recv()
if err != nil {
return fmt.Errorf("connector %q: listen: %w", c.name, err)
}
events, err := c.events(resp.GetEvents())
if err != nil {
return err
}
if err := store(events, resp.GetCursor()); err != nil {
return err
}
}
}
// TargetFacts is what a connector's system of record knows about a target.
// A zero field is a fact it doesn't hold.
type TargetFacts struct {
// OwnerIdentity is who answers for the target, such as "servicenow:jdoe".
OwnerIdentity string
DataClass string
Count int
}
// TargetFacts asks the connector what it knows about t. A connector that
// doesn't serve target facts, or has no record of t, supplies none.
func (c *Connector) TargetFacts(ctx context.Context, t Target) (TargetFacts, error) {
if !c.facts {
return TargetFacts{}, nil
}
req := &pb.TargetFactsRequest{Target: &pb.Target{
Kind: t.Kind, Id: t.ID, Environment: t.Environment, Disposable: t.Disposable,
}}
resp, err := c.client.TargetFacts(ctx, req)
recordProto(ctx, c.name, "target_facts", req, resp, err)
if err != nil {
return TargetFacts{}, fmt.Errorf("connector %q: target facts: %w", c.name, err)
}
if !resp.GetFound() {
return TargetFacts{}, nil
}
return TargetFacts{
OwnerIdentity: resp.GetOwnerIdentity(), DataClass: resp.GetDataClass(), Count: int(resp.GetCount()),
}, nil
}
// Declarations lists the connector's lookups and actions in name order.
func (c *Connector) Declarations() []Declaration {
out := make([]Declaration, 0, len(c.lookups)+len(c.actions))
for _, name := range slices.Sorted(maps.Keys(c.lookups)) {
l := c.lookups[name]
out = append(out, Declaration{
Connector: c.name, Kind: KindLookup, Name: name, Description: l.description, Schema: l.schema, ReadOnly: true,
})
}
for _, name := range slices.Sorted(maps.Keys(c.actions)) {
a := c.actions[name]
out = append(out, Declaration{
Connector: c.name, Kind: KindAction, Name: name, Description: a.description, Schema: a.schema, ReadOnly: a.effect.ReadOnly,
})
}
return out
}
// Name is the connector's accepted name.
func (c *Connector) Name() string { return c.name }
// Version is the connector's own release, as it described itself.
func (c *Connector) Version() string { return c.version }
// SettingsSchema is what the connector declared an operator enters.
func (c *Connector) SettingsSchema() []Setting { return c.settings }
// Parse turns a payload the caller has already verified into typed events,
// each checked against its declared schema.
func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]string) ([]Event, error) {
req := &pb.ParseRequest{Body: body, Headers: headers}
resp, err := c.client.Parse(ctx, req)
recordProto(ctx, c.name, "parse", req, resp, err)
if err != nil {
return nil, fmt.Errorf("connector %q: parse: %w", c.name, err)
}
return c.events(resp.GetEvents())
}
// events checks each wire event against its declared schema.
func (c *Connector) events(wire []*pb.Event) ([]Event, error) {
events := make([]Event, 0, len(wire))
for _, e := range wire {
schema, ok := c.eventTypes[e.GetType()]
if !ok {
return nil, fmt.Errorf("connector %q: event type %q: %w", c.name, e.GetType(), ErrUnknownName)
}
if err := validate(schema, e.GetFields()); err != nil {
return nil, fmt.Errorf("connector %q: event %q: %w", c.name, e.GetType(), err)
}
events = append(events, Event{
Type: e.GetType(), SourceEventID: e.GetSourceEventId(), Fields: e.GetFields(),
LinkKey: e.GetLinkKey(), OccurredAt: e.GetOccurredAt().AsTime(),
})
}
return events, nil
}
// Resolve returns the target and effect of one concrete action.
func (c *Connector) Resolve(ctx context.Context, name string, input jsontext.Value) (Resolution, error) {
if _, err := c.action(name, input); err != nil {
return Resolution{}, err
}
req := &pb.ResolveRequest{Action: name, Input: input}
resp, err := c.client.Resolve(ctx, req)
recordProto(ctx, c.name, "resolve", req, resp, err)
if err != nil {
return Resolution{}, fmt.Errorf("connector %q: resolve %q: %w", c.name, name, err)
}
t := resp.GetTarget()
return Resolution{
Target: Target{Kind: t.GetKind(), ID: t.GetId(), Environment: t.GetEnvironment(), Disposable: t.GetDisposable()},
Effect: effectFrom(resp.GetEffect()),
}, nil
}
// Invoke runs one judged action and returns its output verbatim. The
// connector must report the digest of what it ran; a different digest is
// refused.
func (c *Connector) Invoke(ctx context.Context, inv Invocation) (string, error) {
a, err := c.action(inv.Action, inv.Input)
if err != nil {
return "", err
}
if !a.effect.ReadOnly && !a.idempotent {
return "", fmt.Errorf("connector %q: action %q: %w", c.name, inv.Action, ErrNotIdempotent)
}
inv.Input = Canonical(inv.Input)
digest := Digest(inv.Action, inv.Input)
req := &pb.InvokeRequest{
Action: inv.Action, Input: inv.Input, IdempotencyKey: inv.IdempotencyKey, Digest: digest,
}
resp, err := c.client.Invoke(ctx, req)
recordProto(ctx, c.name, "invoke", req, resp, err)
if err != nil {
return "", fmt.Errorf("connector %q: invoke %q: %w", c.name, inv.Action, err)
}
if resp.GetExecutedDigest() != digest {
return "", fmt.Errorf("connector %q: action %q: %w", c.name, inv.Action, ErrDigestMismatch)
}
if resp.GetOutcomeUnknown() {
return "", fmt.Errorf("connector %q: action %q: %w: %s", c.name, inv.Action, ErrOutcomeUnknown, resp.GetOutput())
}
return string(resp.GetOutput()), nil
}
// Lookup answers a read-only question and returns the answer verbatim. found
// is false when the connector found nothing, which callers report as a finding.
func (c *Connector) Lookup(ctx context.Context, name string, input jsontext.Value) (string, bool, error) {
l, ok := c.lookups[name]
if !ok {
return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(l.input, input); err != nil {
return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
}
req := &pb.LookupRequest{Lookup: name, Input: input}
resp, err := c.client.Lookup(ctx, req)
recordProto(ctx, c.name, "lookup", req, resp, err)
if err != nil {
return "", false, fmt.Errorf("connector %q: lookup %q: %w", c.name, name, err)
}
return string(resp.GetOutput()), resp.GetFound(), nil
}
func (c *Connector) action(name string, input jsontext.Value) (action, error) {
a, ok := c.actions[name]
if !ok {
return action{}, fmt.Errorf("connector %q: action %q: %w", c.name, name, ErrUnknownName)
}
if err := validate(a.input, input); err != nil {
return action{}, fmt.Errorf("connector %q: action %q: %w", c.name, name, err)
}
return a, nil
}
// Digest identifies one concrete action: its name and its exact input. The
// connector SDK computes the same digest over what it ran.
func Digest(action string, input jsontext.Value) string {
h := sha256.New()
h.Write([]byte(action))
h.Write([]byte{0})
h.Write(Canonical(input))
return hex.EncodeToString(h.Sum(nil))
}
// Canonical is the form an input is hashed and sent in. Two spellings of the
// same JSON are the same action, so the bytes mustn't depend on spacing or key
// order: an input that has been through a step record or a decision body comes
// back spelled differently, and an action must still run as the one judged.
// RFC 8785's form keeps a number's exact digits, so two large ids stay two
// actions.
func Canonical(input jsontext.Value) jsontext.Value {
out := slices.Clone(input)
if err := out.Canonicalize(jsontext.CanonicalizeRawInts(false)); err != nil {
return input
}
return out
}
func effectFrom(e *pb.Effect) Effect {
return Effect{ReadOnly: e.GetReadOnly(), Reversibility: Reversibility(e.GetReversibility()), Summary: e.GetSummary()}
}
func schemaOrEmpty(raw []byte) jsontext.Value {
if len(raw) == 0 {
return jsontext.Value(`{}`)
}
return jsontext.Value(raw)
}
func compile(raw []byte) (*jsonschema.Resolved, error) {
if len(raw) == 0 {
raw = []byte(`{}`)
}
var s jsonschema.Schema
if err := json.Unmarshal(raw, &s); err != nil {
return nil, fmt.Errorf("schema: %w", err)
}
resolved, err := s.Resolve(nil)
if err != nil {
return nil, fmt.Errorf("schema: %w", err)
}
return resolved, nil
}
func validate(schema *jsonschema.Resolved, raw jsontext.Value) error {
if len(raw) == 0 {
raw = jsontext.Value(`{}`)
}
var instance any
if err := json.Unmarshal(raw, &instance); err != nil {
return fmt.Errorf("payload is not JSON: %w", err)
}
if err := schema.Validate(instance); err != nil {
return fmt.Errorf("payload doesn't match its schema: %w", err)
}
return nil
}