From 7903a0e225b452f871738075e748e2fb6a0b4e45 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Mon, 16 Nov 2026 04:22:52 +1100 Subject: [PATCH] feat(connectors,deliveries,events): channels are connectors that deliver and listen; messages retry from their rows, and answers arrive as events (archie #1984, #1985) --- internal/app/archie/run.go | 8 +- .../contracts/connector/v1/connector.pb.go | 494 ++++++++++++++---- .../connector/v1/connector_grpc.pb.go | 98 +++- .../migrations/00014_deliveries_channels.sql | 28 + .../migrations/00015_events_source_cursor.sql | 7 + modules/approvals/approvals.go | 16 +- modules/connectors/connector.go | 92 +++- modules/connectors/module.go | 14 + modules/deliveries/deliveries.go | 104 +++- modules/deliveries/internal/store/store.go | 63 ++- modules/events/events.go | 118 ++++- modules/events/internal/store/store.go | 23 + proto/connector/v1/connector.proto | 38 ++ 13 files changed, 975 insertions(+), 128 deletions(-) create mode 100644 internal/platform/pg/migrations/00014_deliveries_channels.sql create mode 100644 internal/platform/pg/migrations/00015_events_source_cursor.sql diff --git a/internal/app/archie/run.go b/internal/app/archie/run.go index 0dc3b5ca3..37664267a 100644 --- a/internal/app/archie/run.go +++ b/internal/app/archie/run.go @@ -77,7 +77,9 @@ func Run(ctx context.Context, configPath string) error { conns.Routes(mux, guard) live := hub.New() - tell := deliveries.NewModule(db, log.With("module", "deliveries"), orgID, live) + // Messages go to the composer and to every accepted connector that delivers + // them. + tell := deliveries.NewModule(db, log.With("module", "deliveries"), orgID, live, conns) tell.Routes(mux, guard) engine := runs.New(db, log.With("module", "runs"), runs.Options{Owner: owner()}) @@ -91,7 +93,8 @@ func Run(ctx context.Context, configPath string) error { allow.Routes(mux, guard) engine.Register("investigate", runs.Investigate(settings, conns, judge, allow, tell)) //nolint:contextcheck // see above - hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine) + // An answer a channel delivers is an event, and settles its request. + hear := events.NewModule(db, log.With("module", "events"), orgID, conns, engine, allow) hear.Routes(mux, guard) engine.Routes(mux, guard, orgID, hear, judge) dashboard.Routes(mux, ui.DistDirFS) @@ -101,6 +104,7 @@ func Run(ctx context.Context, configPath string) error { var workers sync.WaitGroup workers.Go(func() { engine.Work(ctx) }) workers.Go(func() { hear.Work(ctx) }) + workers.Go(func() { tell.Work(ctx) }) err = serve(ctx, log, cfg.Listen, mux) stop() diff --git a/internal/contracts/connector/v1/connector.pb.go b/internal/contracts/connector/v1/connector.pb.go index 4f98de133..353180801 100644 --- a/internal/contracts/connector/v1/connector.pb.go +++ b/internal/contracts/connector/v1/connector.pb.go @@ -204,10 +204,13 @@ func (*DescribeRequest) Descriptor() ([]byte, []int) { type DescribeResponse struct { state protoimpl.MessageState `protogen:"open.v1"` // contract is the contract version the connector speaks, "v1". - Contract string `protobuf:"bytes,1,opt,name=contract,proto3" json:"contract,omitempty"` - Events []*EventType `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` - Actions []*ActionType `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"` - Lookups []*LookupType `protobuf:"bytes,4,rep,name=lookups,proto3" json:"lookups,omitempty"` + Contract string `protobuf:"bytes,1,opt,name=contract,proto3" json:"contract,omitempty"` + Events []*EventType `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` + Actions []*ActionType `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"` + Lookups []*LookupType `protobuf:"bytes,4,rep,name=lookups,proto3" json:"lookups,omitempty"` + Deliveries []*DeliveryType `protobuf:"bytes,5,rep,name=deliveries,proto3" json:"deliveries,omitempty"` + // listens says the connector serves Listen. + Listens bool `protobuf:"varint,6,opt,name=listens,proto3" json:"listens,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -270,6 +273,20 @@ func (x *DescribeResponse) GetLookups() []*LookupType { return nil } +func (x *DescribeResponse) GetDeliveries() []*DeliveryType { + if x != nil { + return x.Deliveries + } + return nil +} + +func (x *DescribeResponse) GetListens() bool { + if x != nil { + return x.Listens + } + return false +} + type EventType struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -412,6 +429,66 @@ func (x *ActionType) GetIdempotent() bool { return false } +type DeliveryType struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"` + InputSchema []byte `protobuf:"bytes,3,opt,name=input_schema,json=inputSchema,proto3" json:"input_schema,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeliveryType) Reset() { + *x = DeliveryType{} + mi := &file_connector_v1_connector_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeliveryType) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeliveryType) ProtoMessage() {} + +func (x *DeliveryType) ProtoReflect() protoreflect.Message { + mi := &file_connector_v1_connector_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeliveryType.ProtoReflect.Descriptor instead. +func (*DeliveryType) Descriptor() ([]byte, []int) { + return file_connector_v1_connector_proto_rawDescGZIP(), []int{6} +} + +func (x *DeliveryType) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DeliveryType) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *DeliveryType) GetInputSchema() []byte { + if x != nil { + return x.InputSchema + } + return nil +} + type LookupType struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -423,7 +500,7 @@ type LookupType struct { func (x *LookupType) Reset() { *x = LookupType{} - mi := &file_connector_v1_connector_proto_msgTypes[6] + mi := &file_connector_v1_connector_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -435,7 +512,7 @@ func (x *LookupType) String() string { func (*LookupType) ProtoMessage() {} func (x *LookupType) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[6] + mi := &file_connector_v1_connector_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -448,7 +525,7 @@ func (x *LookupType) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupType.ProtoReflect.Descriptor instead. func (*LookupType) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{6} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{7} } func (x *LookupType) GetName() string { @@ -485,7 +562,7 @@ type Effect struct { func (x *Effect) Reset() { *x = Effect{} - mi := &file_connector_v1_connector_proto_msgTypes[7] + mi := &file_connector_v1_connector_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -497,7 +574,7 @@ func (x *Effect) String() string { func (*Effect) ProtoMessage() {} func (x *Effect) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[7] + mi := &file_connector_v1_connector_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -510,7 +587,7 @@ func (x *Effect) ProtoReflect() protoreflect.Message { // Deprecated: Use Effect.ProtoReflect.Descriptor instead. func (*Effect) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{7} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{8} } func (x *Effect) GetReadOnly() bool { @@ -548,7 +625,7 @@ type Target struct { func (x *Target) Reset() { *x = Target{} - mi := &file_connector_v1_connector_proto_msgTypes[8] + mi := &file_connector_v1_connector_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -560,7 +637,7 @@ func (x *Target) String() string { func (*Target) ProtoMessage() {} func (x *Target) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[8] + mi := &file_connector_v1_connector_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -573,7 +650,7 @@ func (x *Target) ProtoReflect() protoreflect.Message { // Deprecated: Use Target.ProtoReflect.Descriptor instead. func (*Target) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{8} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{9} } func (x *Target) GetKind() string { @@ -614,7 +691,7 @@ type ParseRequest struct { func (x *ParseRequest) Reset() { *x = ParseRequest{} - mi := &file_connector_v1_connector_proto_msgTypes[9] + mi := &file_connector_v1_connector_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -626,7 +703,7 @@ func (x *ParseRequest) String() string { func (*ParseRequest) ProtoMessage() {} func (x *ParseRequest) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[9] + mi := &file_connector_v1_connector_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -639,7 +716,7 @@ func (x *ParseRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ParseRequest.ProtoReflect.Descriptor instead. func (*ParseRequest) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{9} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{10} } func (x *ParseRequest) GetBody() []byte { @@ -665,7 +742,7 @@ type ParseResponse struct { func (x *ParseResponse) Reset() { *x = ParseResponse{} - mi := &file_connector_v1_connector_proto_msgTypes[10] + mi := &file_connector_v1_connector_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -677,7 +754,7 @@ func (x *ParseResponse) String() string { func (*ParseResponse) ProtoMessage() {} func (x *ParseResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[10] + mi := &file_connector_v1_connector_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -690,7 +767,7 @@ func (x *ParseResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ParseResponse.ProtoReflect.Descriptor instead. func (*ParseResponse) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{10} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{11} } func (x *ParseResponse) GetEvents() []*Event { @@ -717,7 +794,7 @@ type Event struct { func (x *Event) Reset() { *x = Event{} - mi := &file_connector_v1_connector_proto_msgTypes[11] + mi := &file_connector_v1_connector_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -729,7 +806,7 @@ func (x *Event) String() string { func (*Event) ProtoMessage() {} func (x *Event) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[11] + mi := &file_connector_v1_connector_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -742,7 +819,7 @@ func (x *Event) ProtoReflect() protoreflect.Message { // Deprecated: Use Event.ProtoReflect.Descriptor instead. func (*Event) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{11} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{12} } func (x *Event) GetType() string { @@ -790,7 +867,7 @@ type ResolveRequest struct { func (x *ResolveRequest) Reset() { *x = ResolveRequest{} - mi := &file_connector_v1_connector_proto_msgTypes[12] + mi := &file_connector_v1_connector_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -802,7 +879,7 @@ func (x *ResolveRequest) String() string { func (*ResolveRequest) ProtoMessage() {} func (x *ResolveRequest) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[12] + mi := &file_connector_v1_connector_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -815,7 +892,7 @@ func (x *ResolveRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolveRequest.ProtoReflect.Descriptor instead. func (*ResolveRequest) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{12} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{13} } func (x *ResolveRequest) GetAction() string { @@ -842,7 +919,7 @@ type ResolveResponse struct { func (x *ResolveResponse) Reset() { *x = ResolveResponse{} - mi := &file_connector_v1_connector_proto_msgTypes[13] + mi := &file_connector_v1_connector_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -854,7 +931,7 @@ func (x *ResolveResponse) String() string { func (*ResolveResponse) ProtoMessage() {} func (x *ResolveResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[13] + mi := &file_connector_v1_connector_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -867,7 +944,7 @@ func (x *ResolveResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResolveResponse.ProtoReflect.Descriptor instead. func (*ResolveResponse) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{13} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{14} } func (x *ResolveResponse) GetTarget() *Target { @@ -897,7 +974,7 @@ type InvokeRequest struct { func (x *InvokeRequest) Reset() { *x = InvokeRequest{} - mi := &file_connector_v1_connector_proto_msgTypes[14] + mi := &file_connector_v1_connector_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -909,7 +986,7 @@ func (x *InvokeRequest) String() string { func (*InvokeRequest) ProtoMessage() {} func (x *InvokeRequest) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[14] + mi := &file_connector_v1_connector_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -922,7 +999,7 @@ func (x *InvokeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use InvokeRequest.ProtoReflect.Descriptor instead. func (*InvokeRequest) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{14} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{15} } func (x *InvokeRequest) GetAction() string { @@ -967,7 +1044,7 @@ type InvokeResponse struct { func (x *InvokeResponse) Reset() { *x = InvokeResponse{} - mi := &file_connector_v1_connector_proto_msgTypes[15] + mi := &file_connector_v1_connector_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -979,7 +1056,7 @@ func (x *InvokeResponse) String() string { func (*InvokeResponse) ProtoMessage() {} func (x *InvokeResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[15] + mi := &file_connector_v1_connector_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -992,7 +1069,7 @@ func (x *InvokeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use InvokeResponse.ProtoReflect.Descriptor instead. func (*InvokeResponse) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{15} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{16} } func (x *InvokeResponse) GetOutput() []byte { @@ -1026,7 +1103,7 @@ type LookupRequest struct { func (x *LookupRequest) Reset() { *x = LookupRequest{} - mi := &file_connector_v1_connector_proto_msgTypes[16] + mi := &file_connector_v1_connector_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1038,7 +1115,7 @@ func (x *LookupRequest) String() string { func (*LookupRequest) ProtoMessage() {} func (x *LookupRequest) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[16] + mi := &file_connector_v1_connector_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1051,7 +1128,7 @@ func (x *LookupRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupRequest.ProtoReflect.Descriptor instead. func (*LookupRequest) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{16} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{17} } func (x *LookupRequest) GetLookup() string { @@ -1078,7 +1155,7 @@ type LookupResponse struct { func (x *LookupResponse) Reset() { *x = LookupResponse{} - mi := &file_connector_v1_connector_proto_msgTypes[17] + mi := &file_connector_v1_connector_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1090,7 +1167,7 @@ func (x *LookupResponse) String() string { func (*LookupResponse) ProtoMessage() {} func (x *LookupResponse) ProtoReflect() protoreflect.Message { - mi := &file_connector_v1_connector_proto_msgTypes[17] + mi := &file_connector_v1_connector_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1103,7 +1180,7 @@ func (x *LookupResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use LookupResponse.ProtoReflect.Descriptor instead. func (*LookupResponse) Descriptor() ([]byte, []int) { - return file_connector_v1_connector_proto_rawDescGZIP(), []int{17} + return file_connector_v1_connector_proto_rawDescGZIP(), []int{18} } func (x *LookupResponse) GetOutput() []byte { @@ -1120,6 +1197,208 @@ func (x *LookupResponse) GetFound() bool { return false } +type DeliverRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Delivery string `protobuf:"bytes,1,opt,name=delivery,proto3" json:"delivery,omitempty"` + Input []byte `protobuf:"bytes,2,opt,name=input,proto3" json:"input,omitempty"` + IdempotencyKey string `protobuf:"bytes,3,opt,name=idempotency_key,json=idempotencyKey,proto3" json:"idempotency_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeliverRequest) Reset() { + *x = DeliverRequest{} + mi := &file_connector_v1_connector_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeliverRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeliverRequest) ProtoMessage() {} + +func (x *DeliverRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_v1_connector_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeliverRequest.ProtoReflect.Descriptor instead. +func (*DeliverRequest) Descriptor() ([]byte, []int) { + return file_connector_v1_connector_proto_rawDescGZIP(), []int{19} +} + +func (x *DeliverRequest) GetDelivery() string { + if x != nil { + return x.Delivery + } + return "" +} + +func (x *DeliverRequest) GetInput() []byte { + if x != nil { + return x.Input + } + return nil +} + +func (x *DeliverRequest) GetIdempotencyKey() string { + if x != nil { + return x.IdempotencyKey + } + return "" +} + +type DeliverResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // external_id is what the service called what was delivered, for linking. + ExternalId string `protobuf:"bytes,1,opt,name=external_id,json=externalId,proto3" json:"external_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeliverResponse) Reset() { + *x = DeliverResponse{} + mi := &file_connector_v1_connector_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeliverResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeliverResponse) ProtoMessage() {} + +func (x *DeliverResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_v1_connector_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeliverResponse.ProtoReflect.Descriptor instead. +func (*DeliverResponse) Descriptor() ([]byte, []int) { + return file_connector_v1_connector_proto_rawDescGZIP(), []int{20} +} + +func (x *DeliverResponse) GetExternalId() string { + if x != nil { + return x.ExternalId + } + return "" +} + +type ListenRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // cursor is the last cursor the host stored, empty from the start. + Cursor string `protobuf:"bytes,1,opt,name=cursor,proto3" json:"cursor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListenRequest) Reset() { + *x = ListenRequest{} + mi := &file_connector_v1_connector_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListenRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListenRequest) ProtoMessage() {} + +func (x *ListenRequest) ProtoReflect() protoreflect.Message { + mi := &file_connector_v1_connector_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListenRequest.ProtoReflect.Descriptor instead. +func (*ListenRequest) Descriptor() ([]byte, []int) { + return file_connector_v1_connector_proto_rawDescGZIP(), []int{21} +} + +func (x *ListenRequest) GetCursor() string { + if x != nil { + return x.Cursor + } + return "" +} + +type ListenResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*Event `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + Cursor string `protobuf:"bytes,2,opt,name=cursor,proto3" json:"cursor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListenResponse) Reset() { + *x = ListenResponse{} + mi := &file_connector_v1_connector_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListenResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListenResponse) ProtoMessage() {} + +func (x *ListenResponse) ProtoReflect() protoreflect.Message { + mi := &file_connector_v1_connector_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListenResponse.ProtoReflect.Descriptor instead. +func (*ListenResponse) Descriptor() ([]byte, []int) { + return file_connector_v1_connector_proto_rawDescGZIP(), []int{22} +} + +func (x *ListenResponse) GetEvents() []*Event { + if x != nil { + return x.Events + } + return nil +} + +func (x *ListenResponse) GetCursor() string { + if x != nil { + return x.Cursor + } + return "" +} + var File_connector_v1_connector_proto protoreflect.FileDescriptor const file_connector_v1_connector_proto_rawDesc = "" + @@ -1135,12 +1414,16 @@ const file_connector_v1_connector_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x13\n" + "\x11ConfigureResponse\"\x11\n" + - "\x0fDescribeRequest\"\xc7\x01\n" + + "\x0fDescribeRequest\"\x9d\x02\n" + "\x10DescribeResponse\x12\x1a\n" + "\bcontract\x18\x01 \x01(\tR\bcontract\x12/\n" + "\x06events\x18\x02 \x03(\v2\x17.connector.v1.EventTypeR\x06events\x122\n" + "\aactions\x18\x03 \x03(\v2\x18.connector.v1.ActionTypeR\aactions\x122\n" + - "\alookups\x18\x04 \x03(\v2\x18.connector.v1.LookupTypeR\alookups\"Y\n" + + "\alookups\x18\x04 \x03(\v2\x18.connector.v1.LookupTypeR\alookups\x12:\n" + + "\n" + + "deliveries\x18\x05 \x03(\v2\x1a.connector.v1.DeliveryTypeR\n" + + "deliveries\x12\x18\n" + + "\alistens\x18\x06 \x01(\bR\alistens\"Y\n" + "\tEventType\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + "\vdescription\x18\x02 \x01(\tR\vdescription\x12\x16\n" + @@ -1153,7 +1436,11 @@ const file_connector_v1_connector_proto_rawDesc = "" + "\x06effect\x18\x04 \x01(\v2\x14.connector.v1.EffectR\x06effect\x12\x1e\n" + "\n" + "idempotent\x18\x05 \x01(\bR\n" + - "idempotent\"e\n" + + "idempotent\"g\n" + + "\fDeliveryType\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + + "\vdescription\x18\x02 \x01(\tR\vdescription\x12!\n" + + "\finput_schema\x18\x03 \x01(\fR\vinputSchema\"e\n" + "\n" + "LookupType\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12 \n" + @@ -1205,19 +1492,33 @@ const file_connector_v1_connector_proto_rawDesc = "" + "\x05input\x18\x02 \x01(\fR\x05input\">\n" + "\x0eLookupResponse\x12\x16\n" + "\x06output\x18\x01 \x01(\fR\x06output\x12\x14\n" + - "\x05found\x18\x02 \x01(\bR\x05found*\x95\x01\n" + + "\x05found\x18\x02 \x01(\bR\x05found\"k\n" + + "\x0eDeliverRequest\x12\x1a\n" + + "\bdelivery\x18\x01 \x01(\tR\bdelivery\x12\x14\n" + + "\x05input\x18\x02 \x01(\fR\x05input\x12'\n" + + "\x0fidempotency_key\x18\x03 \x01(\tR\x0eidempotencyKey\"2\n" + + "\x0fDeliverResponse\x12\x1f\n" + + "\vexternal_id\x18\x01 \x01(\tR\n" + + "externalId\"'\n" + + "\rListenRequest\x12\x16\n" + + "\x06cursor\x18\x01 \x01(\tR\x06cursor\"U\n" + + "\x0eListenResponse\x12+\n" + + "\x06events\x18\x01 \x03(\v2\x13.connector.v1.EventR\x06events\x12\x16\n" + + "\x06cursor\x18\x02 \x01(\tR\x06cursor*\x95\x01\n" + "\rReversibility\x12\x1d\n" + "\x19REVERSIBILITY_UNSPECIFIED\x10\x00\x12&\n" + "\"REVERSIBILITY_UNDONE_AUTOMATICALLY\x10\x01\x12\x1d\n" + "\x19REVERSIBILITY_RECOVERABLE\x10\x02\x12\x1e\n" + - "\x1aREVERSIBILITY_IRREVERSIBLE\x10\x032\xbf\x03\n" + + "\x1aREVERSIBILITY_IRREVERSIBLE\x10\x032\xce\x04\n" + "\x10ConnectorService\x12L\n" + "\tConfigure\x12\x1e.connector.v1.ConfigureRequest\x1a\x1f.connector.v1.ConfigureResponse\x12I\n" + "\bDescribe\x12\x1d.connector.v1.DescribeRequest\x1a\x1e.connector.v1.DescribeResponse\x12@\n" + "\x05Parse\x12\x1a.connector.v1.ParseRequest\x1a\x1b.connector.v1.ParseResponse\x12F\n" + "\aResolve\x12\x1c.connector.v1.ResolveRequest\x1a\x1d.connector.v1.ResolveResponse\x12C\n" + "\x06Invoke\x12\x1b.connector.v1.InvokeRequest\x1a\x1c.connector.v1.InvokeResponse\x12C\n" + - "\x06Lookup\x12\x1b.connector.v1.LookupRequest\x1a\x1c.connector.v1.LookupResponseB\xc4\x01\n" + + "\x06Lookup\x12\x1b.connector.v1.LookupRequest\x1a\x1c.connector.v1.LookupResponse\x12F\n" + + "\aDeliver\x12\x1c.connector.v1.DeliverRequest\x1a\x1d.connector.v1.DeliverResponse\x12E\n" + + "\x06Listen\x12\x1b.connector.v1.ListenRequest\x1a\x1c.connector.v1.ListenResponse0\x01B\xc4\x01\n" + "\x10com.connector.v1B\x0eConnectorProtoP\x01ZOgithub.com/samcharles93/archie-core/internal/contracts/connector/v1;connectorv1\xa2\x02\x03CXX\xaa\x02\fConnector.V1\xca\x02\fConnector\\V1\xe2\x02\x18Connector\\V1\\GPBMetadata\xea\x02\rConnector::V1b\x06proto3" var ( @@ -1233,7 +1534,7 @@ func file_connector_v1_connector_proto_rawDescGZIP() []byte { } var file_connector_v1_connector_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_connector_v1_connector_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_connector_v1_connector_proto_msgTypes = make([]protoimpl.MessageInfo, 26) var file_connector_v1_connector_proto_goTypes = []any{ (Reversibility)(0), // 0: connector.v1.Reversibility (*ConfigureRequest)(nil), // 1: connector.v1.ConfigureRequest @@ -1242,53 +1543,64 @@ var file_connector_v1_connector_proto_goTypes = []any{ (*DescribeResponse)(nil), // 4: connector.v1.DescribeResponse (*EventType)(nil), // 5: connector.v1.EventType (*ActionType)(nil), // 6: connector.v1.ActionType - (*LookupType)(nil), // 7: connector.v1.LookupType - (*Effect)(nil), // 8: connector.v1.Effect - (*Target)(nil), // 9: connector.v1.Target - (*ParseRequest)(nil), // 10: connector.v1.ParseRequest - (*ParseResponse)(nil), // 11: connector.v1.ParseResponse - (*Event)(nil), // 12: connector.v1.Event - (*ResolveRequest)(nil), // 13: connector.v1.ResolveRequest - (*ResolveResponse)(nil), // 14: connector.v1.ResolveResponse - (*InvokeRequest)(nil), // 15: connector.v1.InvokeRequest - (*InvokeResponse)(nil), // 16: connector.v1.InvokeResponse - (*LookupRequest)(nil), // 17: connector.v1.LookupRequest - (*LookupResponse)(nil), // 18: connector.v1.LookupResponse - nil, // 19: connector.v1.ConfigureRequest.SettingsEntry - nil, // 20: connector.v1.ConfigureRequest.SecretsEntry - nil, // 21: connector.v1.ParseRequest.HeadersEntry - (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*DeliveryType)(nil), // 7: connector.v1.DeliveryType + (*LookupType)(nil), // 8: connector.v1.LookupType + (*Effect)(nil), // 9: connector.v1.Effect + (*Target)(nil), // 10: connector.v1.Target + (*ParseRequest)(nil), // 11: connector.v1.ParseRequest + (*ParseResponse)(nil), // 12: connector.v1.ParseResponse + (*Event)(nil), // 13: connector.v1.Event + (*ResolveRequest)(nil), // 14: connector.v1.ResolveRequest + (*ResolveResponse)(nil), // 15: connector.v1.ResolveResponse + (*InvokeRequest)(nil), // 16: connector.v1.InvokeRequest + (*InvokeResponse)(nil), // 17: connector.v1.InvokeResponse + (*LookupRequest)(nil), // 18: connector.v1.LookupRequest + (*LookupResponse)(nil), // 19: connector.v1.LookupResponse + (*DeliverRequest)(nil), // 20: connector.v1.DeliverRequest + (*DeliverResponse)(nil), // 21: connector.v1.DeliverResponse + (*ListenRequest)(nil), // 22: connector.v1.ListenRequest + (*ListenResponse)(nil), // 23: connector.v1.ListenResponse + nil, // 24: connector.v1.ConfigureRequest.SettingsEntry + nil, // 25: connector.v1.ConfigureRequest.SecretsEntry + nil, // 26: connector.v1.ParseRequest.HeadersEntry + (*timestamppb.Timestamp)(nil), // 27: google.protobuf.Timestamp } var file_connector_v1_connector_proto_depIdxs = []int32{ - 19, // 0: connector.v1.ConfigureRequest.settings:type_name -> connector.v1.ConfigureRequest.SettingsEntry - 20, // 1: connector.v1.ConfigureRequest.secrets:type_name -> connector.v1.ConfigureRequest.SecretsEntry + 24, // 0: connector.v1.ConfigureRequest.settings:type_name -> connector.v1.ConfigureRequest.SettingsEntry + 25, // 1: connector.v1.ConfigureRequest.secrets:type_name -> connector.v1.ConfigureRequest.SecretsEntry 5, // 2: connector.v1.DescribeResponse.events:type_name -> connector.v1.EventType 6, // 3: connector.v1.DescribeResponse.actions:type_name -> connector.v1.ActionType - 7, // 4: connector.v1.DescribeResponse.lookups:type_name -> connector.v1.LookupType - 8, // 5: connector.v1.ActionType.effect:type_name -> connector.v1.Effect - 0, // 6: connector.v1.Effect.reversibility:type_name -> connector.v1.Reversibility - 21, // 7: connector.v1.ParseRequest.headers:type_name -> connector.v1.ParseRequest.HeadersEntry - 12, // 8: connector.v1.ParseResponse.events:type_name -> connector.v1.Event - 22, // 9: connector.v1.Event.occurred_at:type_name -> google.protobuf.Timestamp - 9, // 10: connector.v1.ResolveResponse.target:type_name -> connector.v1.Target - 8, // 11: connector.v1.ResolveResponse.effect:type_name -> connector.v1.Effect - 1, // 12: connector.v1.ConnectorService.Configure:input_type -> connector.v1.ConfigureRequest - 3, // 13: connector.v1.ConnectorService.Describe:input_type -> connector.v1.DescribeRequest - 10, // 14: connector.v1.ConnectorService.Parse:input_type -> connector.v1.ParseRequest - 13, // 15: connector.v1.ConnectorService.Resolve:input_type -> connector.v1.ResolveRequest - 15, // 16: connector.v1.ConnectorService.Invoke:input_type -> connector.v1.InvokeRequest - 17, // 17: connector.v1.ConnectorService.Lookup:input_type -> connector.v1.LookupRequest - 2, // 18: connector.v1.ConnectorService.Configure:output_type -> connector.v1.ConfigureResponse - 4, // 19: connector.v1.ConnectorService.Describe:output_type -> connector.v1.DescribeResponse - 11, // 20: connector.v1.ConnectorService.Parse:output_type -> connector.v1.ParseResponse - 14, // 21: connector.v1.ConnectorService.Resolve:output_type -> connector.v1.ResolveResponse - 16, // 22: connector.v1.ConnectorService.Invoke:output_type -> connector.v1.InvokeResponse - 18, // 23: connector.v1.ConnectorService.Lookup:output_type -> connector.v1.LookupResponse - 18, // [18:24] is the sub-list for method output_type - 12, // [12:18] is the sub-list for method input_type - 12, // [12:12] is the sub-list for extension type_name - 12, // [12:12] is the sub-list for extension extendee - 0, // [0:12] is the sub-list for field type_name + 8, // 4: connector.v1.DescribeResponse.lookups:type_name -> connector.v1.LookupType + 7, // 5: connector.v1.DescribeResponse.deliveries:type_name -> connector.v1.DeliveryType + 9, // 6: connector.v1.ActionType.effect:type_name -> connector.v1.Effect + 0, // 7: connector.v1.Effect.reversibility:type_name -> connector.v1.Reversibility + 26, // 8: connector.v1.ParseRequest.headers:type_name -> connector.v1.ParseRequest.HeadersEntry + 13, // 9: connector.v1.ParseResponse.events:type_name -> connector.v1.Event + 27, // 10: connector.v1.Event.occurred_at:type_name -> google.protobuf.Timestamp + 10, // 11: connector.v1.ResolveResponse.target:type_name -> connector.v1.Target + 9, // 12: connector.v1.ResolveResponse.effect:type_name -> connector.v1.Effect + 13, // 13: connector.v1.ListenResponse.events:type_name -> connector.v1.Event + 1, // 14: connector.v1.ConnectorService.Configure:input_type -> connector.v1.ConfigureRequest + 3, // 15: connector.v1.ConnectorService.Describe:input_type -> connector.v1.DescribeRequest + 11, // 16: connector.v1.ConnectorService.Parse:input_type -> connector.v1.ParseRequest + 14, // 17: connector.v1.ConnectorService.Resolve:input_type -> connector.v1.ResolveRequest + 16, // 18: connector.v1.ConnectorService.Invoke:input_type -> connector.v1.InvokeRequest + 18, // 19: connector.v1.ConnectorService.Lookup:input_type -> connector.v1.LookupRequest + 20, // 20: connector.v1.ConnectorService.Deliver:input_type -> connector.v1.DeliverRequest + 22, // 21: connector.v1.ConnectorService.Listen:input_type -> connector.v1.ListenRequest + 2, // 22: connector.v1.ConnectorService.Configure:output_type -> connector.v1.ConfigureResponse + 4, // 23: connector.v1.ConnectorService.Describe:output_type -> connector.v1.DescribeResponse + 12, // 24: connector.v1.ConnectorService.Parse:output_type -> connector.v1.ParseResponse + 15, // 25: connector.v1.ConnectorService.Resolve:output_type -> connector.v1.ResolveResponse + 17, // 26: connector.v1.ConnectorService.Invoke:output_type -> connector.v1.InvokeResponse + 19, // 27: connector.v1.ConnectorService.Lookup:output_type -> connector.v1.LookupResponse + 21, // 28: connector.v1.ConnectorService.Deliver:output_type -> connector.v1.DeliverResponse + 23, // 29: connector.v1.ConnectorService.Listen:output_type -> connector.v1.ListenResponse + 22, // [22:30] is the sub-list for method output_type + 14, // [14:22] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_connector_v1_connector_proto_init() } @@ -1302,7 +1614,7 @@ func file_connector_v1_connector_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_connector_v1_connector_proto_rawDesc), len(file_connector_v1_connector_proto_rawDesc)), NumEnums: 1, - NumMessages: 21, + NumMessages: 26, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/contracts/connector/v1/connector_grpc.pb.go b/internal/contracts/connector/v1/connector_grpc.pb.go index dea90db04..fc347bec7 100644 --- a/internal/contracts/connector/v1/connector_grpc.pb.go +++ b/internal/contracts/connector/v1/connector_grpc.pb.go @@ -25,6 +25,8 @@ const ( ConnectorService_Resolve_FullMethodName = "/connector.v1.ConnectorService/Resolve" ConnectorService_Invoke_FullMethodName = "/connector.v1.ConnectorService/Invoke" ConnectorService_Lookup_FullMethodName = "/connector.v1.ConnectorService/Lookup" + ConnectorService_Deliver_FullMethodName = "/connector.v1.ConnectorService/Deliver" + ConnectorService_Listen_FullMethodName = "/connector.v1.ConnectorService/Listen" ) // ConnectorServiceClient is the client API for ConnectorService service. @@ -53,6 +55,16 @@ type ConnectorServiceClient interface { // Lookup answers a read-only question. Lookups change nothing and run no // guards; a lookup that finds nothing says so. Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) + // Deliver sends a message or record to people or a system of record. A + // delivery is never offered to an agent: hard rules apply to it, guards + // don't. The host retries a failed delivery, so a connector should make a + // repeat with the same idempotency key harmless where its service allows. + Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error) + // Listen streams the events of a source the connector holds its own + // connection to, such as a chat long poll, from cursor on. Each response + // carries the cursor to resume after its events, which the host keeps only + // once those events are stored. + Listen(ctx context.Context, in *ListenRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ListenResponse], error) } type connectorServiceClient struct { @@ -123,6 +135,35 @@ func (c *connectorServiceClient) Lookup(ctx context.Context, in *LookupRequest, return out, nil } +func (c *connectorServiceClient) Deliver(ctx context.Context, in *DeliverRequest, opts ...grpc.CallOption) (*DeliverResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeliverResponse) + err := c.cc.Invoke(ctx, ConnectorService_Deliver_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *connectorServiceClient) Listen(ctx context.Context, in *ListenRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ListenResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &ConnectorService_ServiceDesc.Streams[0], ConnectorService_Listen_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ListenRequest, ListenResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ConnectorService_ListenClient = grpc.ServerStreamingClient[ListenResponse] + // ConnectorServiceServer is the server API for ConnectorService service. // All implementations must embed UnimplementedConnectorServiceServer // for forward compatibility. @@ -149,6 +190,16 @@ type ConnectorServiceServer interface { // Lookup answers a read-only question. Lookups change nothing and run no // guards; a lookup that finds nothing says so. Lookup(context.Context, *LookupRequest) (*LookupResponse, error) + // Deliver sends a message or record to people or a system of record. A + // delivery is never offered to an agent: hard rules apply to it, guards + // don't. The host retries a failed delivery, so a connector should make a + // repeat with the same idempotency key harmless where its service allows. + Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error) + // Listen streams the events of a source the connector holds its own + // connection to, such as a chat long poll, from cursor on. Each response + // carries the cursor to resume after its events, which the host keeps only + // once those events are stored. + Listen(*ListenRequest, grpc.ServerStreamingServer[ListenResponse]) error mustEmbedUnimplementedConnectorServiceServer() } @@ -177,6 +228,12 @@ func (UnimplementedConnectorServiceServer) Invoke(context.Context, *InvokeReques func (UnimplementedConnectorServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { return nil, status.Error(codes.Unimplemented, "method Lookup not implemented") } +func (UnimplementedConnectorServiceServer) Deliver(context.Context, *DeliverRequest) (*DeliverResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Deliver not implemented") +} +func (UnimplementedConnectorServiceServer) Listen(*ListenRequest, grpc.ServerStreamingServer[ListenResponse]) error { + return status.Error(codes.Unimplemented, "method Listen not implemented") +} func (UnimplementedConnectorServiceServer) mustEmbedUnimplementedConnectorServiceServer() {} func (UnimplementedConnectorServiceServer) testEmbeddedByValue() {} @@ -306,6 +363,35 @@ func _ConnectorService_Lookup_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _ConnectorService_Deliver_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeliverRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ConnectorServiceServer).Deliver(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ConnectorService_Deliver_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ConnectorServiceServer).Deliver(ctx, req.(*DeliverRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ConnectorService_Listen_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(ListenRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(ConnectorServiceServer).Listen(m, &grpc.GenericServerStream[ListenRequest, ListenResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type ConnectorService_ListenServer = grpc.ServerStreamingServer[ListenResponse] + // ConnectorService_ServiceDesc is the grpc.ServiceDesc for ConnectorService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -337,7 +423,17 @@ var ConnectorService_ServiceDesc = grpc.ServiceDesc{ MethodName: "Lookup", Handler: _ConnectorService_Lookup_Handler, }, + { + MethodName: "Deliver", + Handler: _ConnectorService_Deliver_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Listen", + Handler: _ConnectorService_Listen_Handler, + ServerStreams: true, + }, }, - Streams: []grpc.StreamDesc{}, Metadata: "connector/v1/connector.proto", } diff --git a/internal/platform/pg/migrations/00014_deliveries_channels.sql b/internal/platform/pg/migrations/00014_deliveries_channels.sql new file mode 100644 index 000000000..0622b4b12 --- /dev/null +++ b/internal/platform/pg/migrations/00014_deliveries_channels.sql @@ -0,0 +1,28 @@ +-- +goose Up +-- A message goes to the composer and to every accepted connector that delivers +-- messages, one row per channel. A row a connector hasn't confirmed is retried +-- from here, so a channel that is down delays a message rather than losing it. +ALTER TABLE deliveries.messages DROP CONSTRAINT messages_channel_check; +ALTER TABLE deliveries.messages ADD CONSTRAINT messages_channel_check + CHECK (channel ~ '^[a-z][a-z0-9-]{0,62}$'); +ALTER TABLE deliveries.messages + ADD COLUMN attempts int NOT NULL DEFAULT 0, + ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN sent_at timestamptz, + ADD COLUMN external_id text, + ADD COLUMN last_error text; +UPDATE deliveries.messages SET sent_at = created_at; +CREATE INDEX messages_due ON deliveries.messages (next_attempt_at) WHERE sent_at IS NULL; + +-- +goose Down +DROP INDEX deliveries.messages_due; +DELETE FROM deliveries.messages WHERE channel <> 'composer'; +ALTER TABLE deliveries.messages + DROP COLUMN last_error, + DROP COLUMN external_id, + DROP COLUMN sent_at, + DROP COLUMN next_attempt_at, + DROP COLUMN attempts; +ALTER TABLE deliveries.messages DROP CONSTRAINT messages_channel_check; +ALTER TABLE deliveries.messages ADD CONSTRAINT messages_channel_check + CHECK (channel IN ('composer')); diff --git a/internal/platform/pg/migrations/00015_events_source_cursor.sql b/internal/platform/pg/migrations/00015_events_source_cursor.sql new file mode 100644 index 000000000..17efe1b68 --- /dev/null +++ b/internal/platform/pg/migrations/00015_events_source_cursor.sql @@ -0,0 +1,7 @@ +-- +goose Up +-- A source whose connector listens resumes from the cursor after the last +-- events it stored. +ALTER TABLE events.sources ADD COLUMN cursor text NOT NULL DEFAULT ''; + +-- +goose Down +ALTER TABLE events.sources DROP COLUMN cursor; diff --git a/modules/approvals/approvals.go b/modules/approvals/approvals.go index 5aebb9401..0448b9546 100644 --- a/modules/approvals/approvals.go +++ b/modules/approvals/approvals.go @@ -182,11 +182,19 @@ func (m *Module) Answer(ctx context.Context, id uuid.UUID, by, via string, appro return requestOf(row, judgement.Decision{}), nil } -// Settle answers a request on one of the channels that carries it, which is -// what a Telegram tap calls. It reports only whether the answer was taken. -func (m *Module) Settle(ctx context.Context, id uuid.UUID, by, via string, approved bool) error { +// Settle answers a request from an answer a channel delivered as an event. An +// answer that can't settle it, because it was already answered, has expired +// or names no request, reports false rather than an error: retrying the event +// would never change that. +func (m *Module) Settle(ctx context.Context, id uuid.UUID, by, via string, approved bool) (bool, error) { _, err := m.Answer(ctx, id, by, via, approved) - return err + switch { + case err == nil: + return true, nil + case errors.Is(err, ErrAnswered), errors.Is(err, ErrExpired), errors.Is(err, ErrNotFound): + return false, nil + } + return false, err } // Expire settles a request whose deadline passed, so the trail shows why it diff --git a/modules/connectors/connector.go b/modules/connectors/connector.go index 0af367344..63c144a8a 100644 --- a/modules/connectors/connector.go +++ b/modules/connectors/connector.go @@ -127,11 +127,15 @@ type Tool interface { // Connector is a running connector and what it declared. type Connector struct { - name string - client pb.ConnectorServiceClient - events map[string]*jsonschema.Resolved - actions map[string]action - lookups map[string]lookup + name string + client pb.ConnectorServiceClient + eventTypes map[string]*jsonschema.Resolved + actions map[string]action + lookups map[string]lookup + // deliveries are the messages and records it sends, never offered to an + // agent. + deliveries map[string]*jsonschema.Resolved + listens bool } // open configures a connector and reads its declaration. @@ -148,9 +152,11 @@ func open(ctx context.Context, name string, client pb.ConnectorServiceClient, se } c := &Connector{ name: name, client: client, - events: map[string]*jsonschema.Resolved{}, - actions: map[string]action{}, - lookups: map[string]lookup{}, + eventTypes: map[string]*jsonschema.Resolved{}, + actions: map[string]action{}, + lookups: map[string]lookup{}, + deliveries: map[string]*jsonschema.Resolved{}, + listens: desc.GetListens(), } return c, c.load(desc) } @@ -161,7 +167,7 @@ func (c *Connector) load(desc *pb.DescribeResponse) error { if err != nil { return fmt.Errorf("connector %q: event %q: %w", c.name, e.GetName(), err) } - c.events[e.GetName()] = s + c.eventTypes[e.GetName()] = s } for _, a := range desc.GetActions() { s, err := compile(a.GetInputSchema()) @@ -180,9 +186,66 @@ func (c *Connector) load(desc *pb.DescribeResponse) error { } c.lookups[l.GetName()] = lookup{input: s, description: l.GetDescription(), schema: schemaOrEmpty(l.GetInputSchema())} } + for _, d := range desc.GetDeliveries() { + s, err := compile(d.GetInputSchema()) + if err != nil { + return fmt.Errorf("connector %q: delivery %q: %w", c.name, d.GetName(), err) + } + c.deliveries[d.GetName()] = s + } return nil } +// Delivers reports whether the connector sends the named delivery. +func (c *Connector) Delivers(name string) bool { + _, ok := c.deliveries[name] + return ok +} + +// Deliver sends one delivery, checked against its declared schema, and +// returns what the service called it. +func (c *Connector) Deliver(ctx context.Context, name string, input jsontext.Value, key string) (string, error) { + schema, ok := c.deliveries[name] + if !ok { + return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, ErrUnknownName) + } + if err := validate(schema, input); err != nil { + return "", fmt.Errorf("connector %q: delivery %q: %w", c.name, name, err) + } + resp, err := c.client.Deliver(ctx, &pb.DeliverRequest{Delivery: name, Input: input, IdempotencyKey: key}) + if err != nil { + return "", fmt.Errorf("connector %q: deliver %q: %w", c.name, name, err) + } + return resp.GetExternalId(), nil +} + +// Listens reports whether the connector holds its own connection to a source +// and streams its events. +func (c *Connector) Listens() bool { return c.listens } + +// Listen streams the connector's events from cursor on, handing each batch +// and the cursor after it to store. It returns when ctx ends, the stream +// breaks, or store fails; the caller listens again from its last cursor. +func (c *Connector) Listen(ctx context.Context, cursor string, store func(events []Event, cursor string) error) error { + stream, err := c.client.Listen(ctx, &pb.ListenRequest{Cursor: cursor}) + if err != nil { + return fmt.Errorf("connector %q: listen: %w", c.name, err) + } + for { + resp, err := stream.Recv() + if err != nil { + return fmt.Errorf("connector %q: listen: %w", c.name, err) + } + events, err := c.events(resp.GetEvents()) + if err != nil { + return err + } + if err := store(events, resp.GetCursor()); err != nil { + return err + } + } +} + // Declarations lists the connector's lookups and actions in name order. func (c *Connector) Declarations() []Declaration { out := make([]Declaration, 0, len(c.lookups)+len(c.actions)) @@ -211,9 +274,14 @@ func (c *Connector) Parse(ctx context.Context, body []byte, headers map[string]s if err != nil { return nil, fmt.Errorf("connector %q: parse: %w", c.name, err) } - events := make([]Event, 0, len(resp.GetEvents())) - for _, e := range resp.GetEvents() { - schema, ok := c.events[e.GetType()] + return c.events(resp.GetEvents()) +} + +// events checks each wire event against its declared schema. +func (c *Connector) events(wire []*pb.Event) ([]Event, error) { + events := make([]Event, 0, len(wire)) + for _, e := range wire { + schema, ok := c.eventTypes[e.GetType()] if !ok { return nil, fmt.Errorf("connector %q: event type %q: %w", c.name, e.GetType(), ErrUnknownName) } diff --git a/modules/connectors/module.go b/modules/connectors/module.go index 73554cfbf..19a89d00b 100644 --- a/modules/connectors/module.go +++ b/modules/connectors/module.go @@ -121,6 +121,20 @@ func (m *Module) Tool(name string) (Tool, bool) { return s, ok } +// Delivering lists the running connectors that send the named delivery, in +// name order: the channels a message goes out on. +func (m *Module) Delivering(delivery string) []*Connector { + m.mu.RLock() + defer m.mu.RUnlock() + var out []*Connector + for _, name := range slices.Sorted(maps.Keys(m.running)) { + if c := m.running[name]; c.Delivers(delivery) { + out = append(out, c) + } + } + return out +} + // Declarations lists every running connector's lookups and actions and every // MCP server's declared tools, ordered by name and then by tool name. func (m *Module) Declarations() []Declaration { diff --git a/modules/deliveries/deliveries.go b/modules/deliveries/deliveries.go index 312b40865..b69acb390 100644 --- a/modules/deliveries/deliveries.go +++ b/modules/deliveries/deliveries.go @@ -2,12 +2,15 @@ // channel. It holds no approval state: approvals own the request and its // answer, and use deliveries to carry them. The composer, the chat box in the // dashboard, is the first channel: its messages are rows a client reads from a -// cursor, with a live stream that only says when to read. +// cursor, with a live stream that only says when to read. Every accepted +// connector that delivers messages is another: each gets its own row, retried +// until the connector confirms it. package deliveries import ( "context" "database/sql" + json "encoding/json/v2" "errors" "fmt" "log/slog" @@ -20,6 +23,7 @@ import ( "github.com/samcharles93/archie-core/internal/platform/httpjson" "github.com/samcharles93/archie-core/internal/platform/hub" + "github.com/samcharles93/archie-core/modules/connectors" "github.com/samcharles93/archie-core/modules/deliveries/internal/store" ) @@ -28,6 +32,9 @@ const ( topicComposer = "composer" pageSize = 100 heartbeat = 25 * time.Second + // deliveryMessage is the delivery a connector declares to carry messages. + deliveryMessage = "message" + maxBackoff = 10 * time.Minute ) // The kinds of message a channel carries. @@ -77,18 +84,26 @@ type Message struct { CreatedAt time.Time `json:"created_at"` } +// Channels are the running connectors a message can go out on. +type Channels interface { + Delivering(delivery string) []*connectors.Connector + Get(name string) (*connectors.Connector, bool) +} + // Module delivers one organisation's notifications. type Module struct { - orgID string - store *store.Store - hub *hub.Hub - log *slog.Logger + orgID string + store *store.Store + hub *hub.Hub + channels Channels + log *slog.Logger + poll time.Duration } // NewModule returns the deliveries module for orgID, announcing new messages -// on h. -func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub) *Module { - return &Module{orgID: orgID, store: store.New(db), hub: h, log: log} +// on h and sending them on every connector in channels that delivers them. +func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, h *hub.Hub, channels Channels) *Module { + return &Module{orgID: orgID, store: store.New(db), hub: h, channels: channels, log: log, poll: time.Second} } // Notify delivers n on every channel. Delivering the same key again changes @@ -126,9 +141,10 @@ func (m *Module) Told(ctx context.Context, a ApprovalRequest, answer string) err }) } -// send writes msg to every channel and wakes the composer's readers. +// send writes msg to the composer, which wakes its readers, and queues it for +// every connector that delivers messages. Work sends the queued rows. func (m *Module) send(ctx context.Context, msg store.Message) error { - msg.OrgID, msg.Channel = m.orgID, channelComposer + msg.OrgID, msg.Channel, msg.Sent = m.orgID, channelComposer, true seq, created, err := m.store.Insert(ctx, msg) if err != nil { return err @@ -136,9 +152,77 @@ func (m *Module) send(ctx context.Context, msg store.Message) error { if created { m.hub.Publish(hub.Note{Topic: topicComposer, Seq: seq}) } + for _, c := range m.channels.Delivering(deliveryMessage) { + // Each channel keeps its own row, so each carries its own id. + row := msg + row.ID, row.Channel, row.Sent = uuid.New(), c.Name(), false + if _, _, err := m.store.Insert(ctx, row); err != nil { + return err + } + } return nil } +// Work sends queued messages to their connectors until ctx ends. A failed send +// is tried again after a backoff that grows to maxBackoff, never dropped. +func (m *Module) Work(ctx context.Context) { + for ctx.Err() == nil { + msg, found, err := m.store.Claim(ctx, backoff) + if err != nil { + m.log.ErrorContext(ctx, "claim delivery", "error", err) + } + if found { + m.deliver(ctx, msg) + continue + } + select { + case <-ctx.Done(): + case <-time.After(m.poll): + } + } +} + +func (m *Module) deliver(ctx context.Context, msg store.Message) { + external, err := m.sendTo(ctx, msg) + if err != nil { + m.log.WarnContext(ctx, "delivery failed", "channel", msg.Channel, "message", msg.ID, "attempt", msg.Attempts, "error", err) + if err := m.store.Failed(ctx, msg.Seq, err.Error()); err != nil { + m.log.ErrorContext(ctx, "record failed delivery", "error", err) + } + return + } + if err := m.store.Sent(ctx, msg.Seq, external); err != nil { + m.log.ErrorContext(ctx, "record delivery", "error", err) + } +} + +func (m *Module) sendTo(ctx context.Context, msg store.Message) (string, error) { + c, ok := m.channels.Get(msg.Channel) + if !ok { + return "", fmt.Errorf("connector %q isn't running", msg.Channel) + } + input := map[string]string{"title": msg.Title} + if msg.Body != "" { + input["body"] = msg.Body + } + if msg.Kind == KindApprovalRequest && msg.RequestID.Valid { + input["approval_request_id"] = msg.RequestID.UUID.String() + } + raw, err := json.Marshal(input) + if err != nil { + return "", err + } + return c.Deliver(ctx, deliveryMessage, raw, msg.Key) +} + +// backoff is how long to wait after the attempts so far before the next. +func backoff(attempts int) time.Duration { + if attempts >= 10 { + return maxBackoff + } + return min(time.Duration(1< $3) WHERE seq = $1`, + m.Seq, m.Attempts, backoff(m.Attempts).Seconds()); err != nil { + return Message{}, false, fmt.Errorf("lease message: %w", err) + } + if err := tx.Commit(); err != nil { + return Message{}, false, fmt.Errorf("lease message: %w", err) + } + return m, true, nil +} + +// Sent records that a channel confirmed a message. +func (s *Store) Sent(ctx context.Context, seq int64, externalID string) error { + if _, err := s.db.ExecContext(ctx, ` + UPDATE deliveries.messages SET sent_at = now(), external_id = $2, last_error = NULL WHERE seq = $1`, + seq, externalID); err != nil { + return fmt.Errorf("mark message sent: %w", err) + } + return nil +} + +// Failed records why an attempt failed; the lease already set when it is +// tried again. +func (s *Store) Failed(ctx context.Context, seq int64, reason string) error { + if _, err := s.db.ExecContext(ctx, ` + UPDATE deliveries.messages SET last_error = $2 WHERE seq = $1`, seq, reason); err != nil { + return fmt.Errorf("record failed delivery: %w", err) + } + return nil +} diff --git a/modules/events/events.go b/modules/events/events.go index 50da50206..0073e499b 100644 --- a/modules/events/events.go +++ b/modules/events/events.go @@ -1,7 +1,10 @@ // Package events is how Archie hears: sources deliver payloads, a connector // turns each verified payload into typed events, and every event is stored -// before it is acknowledged. A dispatcher then claims each new event, links a -// resolve to what it resolves, and starts a run for every trigger it matches. +// before it is acknowledged. A source whose connector holds its own connection +// (a chat long poll) streams its events instead, resuming from a stored +// cursor. A dispatcher then claims each new event, settles an approval it +// answers, links a resolve to what it resolves, and starts a run for every +// trigger it matches. package events import ( @@ -16,6 +19,7 @@ import ( "net/http" "regexp" "strconv" + "sync" "time" "github.com/google/uuid" @@ -40,19 +44,32 @@ type Runs interface { Start(ctx context.Context, req runs.StartRequest) (uuid.UUID, error) } +// Answers settles the approval requests answer events name. +type Answers interface { + Settle(ctx context.Context, id uuid.UUID, by, via string, approved bool) (bool, error) +} + +// answerType is the event a channel emits when a person answers an approval +// request. +const answerType = "approval.answer" + +// listenRetry is how long a listener waits after its stream breaks. +const listenRetry = 5 * time.Second + // Module hears events for one organisation. type Module struct { orgID string store *store.Store connectors Connectors runs Runs + answers Answers log *slog.Logger poll time.Duration } // NewModule returns the events module for orgID. -func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs) *Module { - return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, log: log, poll: time.Second} +func NewModule(db *sqlx.DB, log *slog.Logger, orgID string, c Connectors, r Runs, a Answers) *Module { + return &Module{orgID: orgID, store: store.New(db), connectors: c, runs: r, answers: a, log: log, poll: time.Second} } // Routes adds the webhook intake and the guarded admin routes. @@ -121,9 +138,18 @@ func (m *Module) save(ctx context.Context, source string, parsed []connectors.Ev return stored, duplicates, nil } -// Work dispatches stored events until ctx ends. +// Work listens to every listening source and dispatches stored events until +// ctx ends. func (m *Module) Work(ctx context.Context) { + var listeners sync.WaitGroup + defer listeners.Wait() + listening := map[string]bool{} + nextCheck := time.Time{} for ctx.Err() == nil { + if time.Now().After(nextCheck) { + m.startListeners(ctx, &listeners, listening) + nextCheck = time.Now().Add(listenRetry) + } found, err := m.store.Dispatch(ctx, m.dispatch) if err != nil { m.log.ErrorContext(ctx, "dispatch event", "error", err) @@ -138,11 +164,93 @@ func (m *Module) Work(ctx context.Context) { } } +// startListeners starts one listener for each source whose connector listens +// and has none yet. A listener lives until ctx ends. +func (m *Module) startListeners(ctx context.Context, wg *sync.WaitGroup, listening map[string]bool) { + sources, err := m.store.Sources(ctx, m.orgID) + if err != nil { + m.log.ErrorContext(ctx, "list sources", "error", err) + return + } + for _, src := range sources { + conn, ok := m.connectors.Get(src.Connector) + if listening[src.Name] || !ok || !conn.Listens() { + continue + } + listening[src.Name] = true + wg.Go(func() { m.listen(ctx, src) }) + } +} + +// listen streams one source's events, storing each batch before its cursor, +// so a restart fetches again anything not yet stored and dedupe drops what +// was. A broken stream is opened again from the stored cursor. +func (m *Module) listen(ctx context.Context, src store.Source) { + cursor := src.Cursor + for ctx.Err() == nil { + conn, ok := m.connectors.Get(src.Connector) + var err error + if !ok { + err = fmt.Errorf("connector %q isn't running", src.Connector) + } else { + err = conn.Listen(ctx, cursor, func(events []connectors.Event, next string) error { + if _, _, err := m.save(ctx, src.Name, events); err != nil { + return err + } + if err := m.store.SetCursor(ctx, m.orgID, src.Name, next); err != nil { + return err + } + cursor = next + return nil + }) + } + if ctx.Err() != nil { + return + } + m.log.WarnContext(ctx, "listening stopped", "source", src.Name, "error", err) + select { + case <-ctx.Done(): + case <-time.After(listenRetry): + } + } +} + +// answer settles the approval request an answer event names. An answer that +// can't settle it is logged and done with: the request's own record already +// says why. +func (m *Module) answer(ctx context.Context, e store.Event) error { + var a struct { + RequestID uuid.UUID `json:"request_id"` + Approved bool `json:"approved"` + By string `json:"by"` + ByName string `json:"by_name"` + } + if err := json.Unmarshal(e.Fields, &a); err != nil { + m.log.WarnContext(ctx, "unreadable approval answer", "event", e.ID, "error", err) + return nil + } + by := a.By + if a.ByName != "" { + by = a.ByName + " (" + a.By + ")" + } + settled, err := m.answers.Settle(ctx, a.RequestID, by, e.Source, a.Approved) + if err != nil { + return fmt.Errorf("settle approval %s: %w", a.RequestID, err) + } + if !settled { + m.log.InfoContext(ctx, "approval answer changed nothing", "event", e.ID, "request", a.RequestID) + } + return nil +} + // dispatch links a resolve to the event it resolves and starts no work for // it. Any other event starts one run per matching trigger, keyed so a retried // dispatch never starts a second run. func (m *Module) dispatch(tx *sqlx.Tx, e store.Event) (bool, error) { ctx := context.Background() + if e.Type == answerType { + return false, m.answer(ctx, e) + } linked, err := store.Link(ctx, tx, e) if err != nil || linked { return false, err diff --git a/modules/events/internal/store/store.go b/modules/events/internal/store/store.go index c35ecb569..17ff52a10 100644 --- a/modules/events/internal/store/store.go +++ b/modules/events/internal/store/store.go @@ -24,6 +24,8 @@ type Source struct { Name string `db:"name"` Connector string `db:"connector"` TokenSHA256 string `db:"token_sha256"` + // Cursor is where a listening source resumes. + Cursor string `db:"cursor"` } // Event is one stored event. @@ -77,6 +79,27 @@ func (s *Store) Source(ctx context.Context, orgID, name string) (Source, error) return src, nil } +// Sources returns every source of the organisation. +func (s *Store) Sources(ctx context.Context, orgID string) ([]Source, error) { + var out []Source + if err := s.db.SelectContext(ctx, &out, ` + SELECT org_id, name, connector, token_sha256, cursor FROM events.sources + WHERE org_id = $1 ORDER BY name`, orgID); err != nil { + return nil, fmt.Errorf("list sources: %w", err) + } + return out, nil +} + +// SetCursor records where a listening source resumes. +func (s *Store) SetCursor(ctx context.Context, orgID, name, cursor string) error { + if _, err := s.db.ExecContext(ctx, ` + UPDATE events.sources SET cursor = $3, updated_at = now() WHERE org_id = $1 AND name = $2`, + orgID, name, cursor); err != nil { + return fmt.Errorf("save cursor: %w", err) + } + return nil +} + // UpsertSource creates a source or replaces its connector and token. func (s *Store) UpsertSource(ctx context.Context, src Source) error { if _, err := s.db.NamedExecContext(ctx, ` diff --git a/proto/connector/v1/connector.proto b/proto/connector/v1/connector.proto index a17da758f..abdaf981e 100644 --- a/proto/connector/v1/connector.proto +++ b/proto/connector/v1/connector.proto @@ -26,6 +26,16 @@ service ConnectorService { // Lookup answers a read-only question. Lookups change nothing and run no // guards; a lookup that finds nothing says so. rpc Lookup(LookupRequest) returns (LookupResponse); + // Deliver sends a message or record to people or a system of record. A + // delivery is never offered to an agent: hard rules apply to it, guards + // don't. The host retries a failed delivery, so a connector should make a + // repeat with the same idempotency key harmless where its service allows. + rpc Deliver(DeliverRequest) returns (DeliverResponse); + // Listen streams the events of a source the connector holds its own + // connection to, such as a chat long poll, from cursor on. Each response + // carries the cursor to resume after its events, which the host keeps only + // once those events are stored. + rpc Listen(ListenRequest) returns (stream ListenResponse); } message ConfigureRequest { @@ -41,6 +51,9 @@ message DescribeResponse { repeated EventType events = 2; repeated ActionType actions = 3; repeated LookupType lookups = 4; + repeated DeliveryType deliveries = 5; + // listens says the connector serves Listen. + bool listens = 6; } message EventType { @@ -63,6 +76,12 @@ message ActionType { bool idempotent = 5; } +message DeliveryType { + string name = 1; + string description = 2; + bytes input_schema = 3; +} + message LookupType { string name = 1; string description = 2; @@ -149,3 +168,22 @@ message LookupResponse { bytes output = 1; bool found = 2; } + +message DeliverRequest { + string delivery = 1; + bytes input = 2; + string idempotency_key = 3; +} +message DeliverResponse { + // external_id is what the service called what was delivered, for linking. + string external_id = 1; +} + +message ListenRequest { + // cursor is the last cursor the host stored, empty from the start. + string cursor = 1; +} +message ListenResponse { + repeated Event events = 1; + string cursor = 2; +}