feat(presence): every service re-stamps a presence record in the State Store
This commit is contained in:
20 files changed
+1461
-660
No files matched your search
@@ -23,11 +23,13 @@ import (
|
||||
natsio "github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/agentexec"
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
@@ -178,6 +180,9 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
if err := b.setupGatewayClient(); err != nil {
|
||||
return 1
|
||||
}
|
||||
if ps, ok := b.stateStore.(storecontract.PresenceStore); ok {
|
||||
go presence.Run(ctx, b.processName, servicekit.Build(), ps, b.healthRegistry, b.log)
|
||||
}
|
||||
// The daemon publishes the configuration view, which carries the catalog's
|
||||
// model limits.
|
||||
b.startModelCatalogRefresh(ctx)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/curator"
|
||||
domainmemory "github.com/samcharles93/archie-core/internal/domain/memory"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
|
||||
@@ -189,6 +190,7 @@ func (b *server) openState(ctx context.Context) error {
|
||||
b.cfg, b.runtimeVersions = cfg, versions
|
||||
b.cfgHolder.Set(cfg)
|
||||
go b.applyStatus.Run(ctx)
|
||||
go presence.Run(ctx, presence.Gateway, servicekit.Build(), client, nil, b.log)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,9 +5,11 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/controlplanerpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
|
||||
@@ -65,6 +67,7 @@ func Run(ctx context.Context, o Options) error {
|
||||
}
|
||||
reporter.Report(ctx, controlplanerpc.ChannelSettingsKind, channelVersion, nil)
|
||||
go reporter.Run(ctx)
|
||||
go presence.Run(ctx, presence.Messaging, servicekit.Build(), stateStore, health, log)
|
||||
cfg.Telegram, cfg.Email, cfg.Webhook, cfg.WebhookAddr, cfg.ShowToolCalls = chatSettings.Telegram, chatSettings.Email, chatSettings.Webhook, chatSettings.WebhookAddr, chatSettings.ShowToolCalls
|
||||
cfg.TelegramToken, settingsErr = resolveTelegramToken(chatSettings.Telegram, secrets)
|
||||
if settingsErr != nil {
|
||||
|
||||
@@ -9,11 +9,14 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/servicekit"
|
||||
"github.com/samcharles93/archie-core/internal/domain/access"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
@@ -108,11 +111,8 @@ func Run(ctx context.Context, options Options) error {
|
||||
// events back out of the State Store over the same cursor SSE catch-up
|
||||
// uses. Backgrounded so priming past a large events table cannot delay the
|
||||
// listener.
|
||||
go func() {
|
||||
if err := srv.PumpEvents(ctx, opts.EventPollInterval); err != nil {
|
||||
log.Error("event pump stopped; the activity feed will only show history", "err", err)
|
||||
}
|
||||
}()
|
||||
go pumpEvents(ctx, srv, opts.EventPollInterval, log)
|
||||
go presence.Run(ctx, presence.UI, servicekit.Build(), tasks, srv.Health, log)
|
||||
liveCtx, stopLive := context.WithCancel(ctx)
|
||||
defer stopLive()
|
||||
go srv.RunLive(liveCtx)
|
||||
@@ -158,3 +158,9 @@ func serve(ctx context.Context, listener net.Listener, handler http.Handler, opt
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func pumpEvents(ctx context.Context, srv *webui.Server, interval time.Duration, log *slog.Logger) {
|
||||
if err := srv.PumpEvents(ctx, interval); err != nil {
|
||||
log.Error("event pump stopped; the activity feed will only show history", "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package servicekit
|
||||
|
||||
import (
|
||||
"github.com/samcharles93/archie-core/internal/buildinfo"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/installtype"
|
||||
)
|
||||
|
||||
// Build is what this binary was compiled as, for its presence record.
|
||||
func Build() presence.Build {
|
||||
return presence.Build{Version: buildinfo.Version, InstallType: installtype.Type()}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
"github.com/samcharles93/archie-core/internal/domain/identity"
|
||||
"github.com/samcharles93/archie-core/internal/domain/org"
|
||||
"github.com/samcharles93/archie-core/internal/domain/presence"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storepkg"
|
||||
infraaccess "github.com/samcharles93/archie-core/internal/infrastructure/access"
|
||||
@@ -338,6 +339,9 @@ func (b *server) stateStoreDeps(grants *staterpc.TaskGrants) staterpc.Deps {
|
||||
if css, ok := b.st.(storecontract.ConfigSnapshotStore); ok {
|
||||
deps.ConfigSnapshots = css
|
||||
}
|
||||
if ps, ok := b.st.(storecontract.PresenceStore); ok {
|
||||
deps.Presence = ps
|
||||
}
|
||||
if as, ok := b.st.(storecontract.ApplyStatusStore); ok {
|
||||
deps.ApplyStatus = as
|
||||
}
|
||||
@@ -406,7 +410,7 @@ func serveStateStore(ctx context.Context, listener net.Listener, deps staterpc.D
|
||||
// and answers 503 when the store is degraded. It is the standalone process's
|
||||
// counterpart to the daemon's /healthz + /health/detailed (internal/webui).
|
||||
func (b *server) startStateStoreReadiness(ctx context.Context, readyAddr string) error {
|
||||
registry := health.NewRegistry(readiness.NewStoreProbe(b.st))
|
||||
registry := b.readinessRegistry()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -427,6 +431,10 @@ func (b *server) startStateStoreReadiness(ctx context.Context, readyAddr string)
|
||||
return b.serveHealth(ctx, readyAddr, mux, "state store readiness")
|
||||
}
|
||||
|
||||
func (b *server) readinessRegistry() *health.Registry {
|
||||
return health.NewRegistry(readiness.NewStoreProbe(b.st))
|
||||
}
|
||||
|
||||
// startOptionalSurfaces starts the HTTP surfaces this process serves beside
|
||||
// the gRPC contract, each enabled only by its own flag.
|
||||
//
|
||||
@@ -435,6 +443,9 @@ func (b *server) startStateStoreReadiness(ctx context.Context, readyAddr string)
|
||||
// own decision, not to the boot sequence that triggers it -- the same reason
|
||||
// reportUnseededResources is a method.
|
||||
func (b *server) startOptionalSurfaces(ctx context.Context, options Options) error {
|
||||
if ps, ok := b.st.(storecontract.PresenceStore); ok {
|
||||
go presence.Run(ctx, presence.StateStore, servicekit.Build(), ps, b.readinessRegistry(), b.log)
|
||||
}
|
||||
if options.ReadyAddr != "" {
|
||||
if err := b.startStateStoreReadiness(ctx, options.ReadyAddr); err != nil {
|
||||
return err
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -75,6 +75,8 @@ const (
|
||||
StateStoreService_ListChannelStatus_FullMethodName = "/state.v1.StateStoreService/ListChannelStatus"
|
||||
StateStoreService_PutApplyStatus_FullMethodName = "/state.v1.StateStoreService/PutApplyStatus"
|
||||
StateStoreService_ListApplyStatus_FullMethodName = "/state.v1.StateStoreService/ListApplyStatus"
|
||||
StateStoreService_PutPresence_FullMethodName = "/state.v1.StateStoreService/PutPresence"
|
||||
StateStoreService_ListPresence_FullMethodName = "/state.v1.StateStoreService/ListPresence"
|
||||
StateStoreService_ReadTaskLog_FullMethodName = "/state.v1.StateStoreService/ReadTaskLog"
|
||||
StateStoreService_StreamTaskLogContent_FullMethodName = "/state.v1.StateStoreService/StreamTaskLogContent"
|
||||
StateStoreService_InsertCapture_FullMethodName = "/state.v1.StateStoreService/InsertCapture"
|
||||
@@ -230,6 +232,11 @@ type StateStoreServiceClient interface {
|
||||
// Both are administrative.
|
||||
PutApplyStatus(ctx context.Context, in *PutApplyStatusRequest, opts ...grpc.CallOption) (*PutApplyStatusResponse, error)
|
||||
ListApplyStatus(ctx context.Context, in *ListApplyStatusRequest, opts ...grpc.CallOption) (*ListApplyStatusResponse, error)
|
||||
// Presence: every running service instance re-stamps one record, so a
|
||||
// record past the staleness window means that instance is down.
|
||||
// Administrative.
|
||||
PutPresence(ctx context.Context, in *PutPresenceRequest, opts ...grpc.CallOption) (*PutPresenceResponse, error)
|
||||
ListPresence(ctx context.Context, in *ListPresenceRequest, opts ...grpc.CallOption) (*ListPresenceResponse, error)
|
||||
// Task log: one task attempt's persisted log. Task log files live in the
|
||||
// state directory this process owns, so the dashboard reads them over this
|
||||
// contract rather than opening them itself
|
||||
@@ -878,6 +885,26 @@ func (c *stateStoreServiceClient) ListApplyStatus(ctx context.Context, in *ListA
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *stateStoreServiceClient) PutPresence(ctx context.Context, in *PutPresenceRequest, opts ...grpc.CallOption) (*PutPresenceResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(PutPresenceResponse)
|
||||
err := c.cc.Invoke(ctx, StateStoreService_PutPresence_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *stateStoreServiceClient) ListPresence(ctx context.Context, in *ListPresenceRequest, opts ...grpc.CallOption) (*ListPresenceResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ListPresenceResponse)
|
||||
err := c.cc.Invoke(ctx, StateStoreService_ListPresence_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *stateStoreServiceClient) ReadTaskLog(ctx context.Context, in *ReadTaskLogRequest, opts ...grpc.CallOption) (*ReadTaskLogResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadTaskLogResponse)
|
||||
@@ -1418,6 +1445,11 @@ type StateStoreServiceServer interface {
|
||||
// Both are administrative.
|
||||
PutApplyStatus(context.Context, *PutApplyStatusRequest) (*PutApplyStatusResponse, error)
|
||||
ListApplyStatus(context.Context, *ListApplyStatusRequest) (*ListApplyStatusResponse, error)
|
||||
// Presence: every running service instance re-stamps one record, so a
|
||||
// record past the staleness window means that instance is down.
|
||||
// Administrative.
|
||||
PutPresence(context.Context, *PutPresenceRequest) (*PutPresenceResponse, error)
|
||||
ListPresence(context.Context, *ListPresenceRequest) (*ListPresenceResponse, error)
|
||||
// Task log: one task attempt's persisted log. Task log files live in the
|
||||
// state directory this process owns, so the dashboard reads them over this
|
||||
// contract rather than opening them itself
|
||||
@@ -1674,6 +1706,12 @@ func (UnimplementedStateStoreServiceServer) PutApplyStatus(context.Context, *Put
|
||||
func (UnimplementedStateStoreServiceServer) ListApplyStatus(context.Context, *ListApplyStatusRequest) (*ListApplyStatusResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ListApplyStatus not implemented")
|
||||
}
|
||||
func (UnimplementedStateStoreServiceServer) PutPresence(context.Context, *PutPresenceRequest) (*PutPresenceResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method PutPresence not implemented")
|
||||
}
|
||||
func (UnimplementedStateStoreServiceServer) ListPresence(context.Context, *ListPresenceRequest) (*ListPresenceResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ListPresence not implemented")
|
||||
}
|
||||
func (UnimplementedStateStoreServiceServer) ReadTaskLog(context.Context, *ReadTaskLogRequest) (*ReadTaskLogResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ReadTaskLog not implemented")
|
||||
}
|
||||
@@ -2823,6 +2861,42 @@ func _StateStoreService_ListApplyStatus_Handler(srv interface{}, ctx context.Con
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _StateStoreService_PutPresence_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(PutPresenceRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(StateStoreServiceServer).PutPresence(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: StateStoreService_PutPresence_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(StateStoreServiceServer).PutPresence(ctx, req.(*PutPresenceRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _StateStoreService_ListPresence_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ListPresenceRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(StateStoreServiceServer).ListPresence(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: StateStoreService_ListPresence_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(StateStoreServiceServer).ListPresence(ctx, req.(*ListPresenceRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _StateStoreService_ReadTaskLog_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadTaskLogRequest)
|
||||
if err := dec(in); err != nil {
|
||||
@@ -3753,6 +3827,14 @@ var StateStoreService_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "ListApplyStatus",
|
||||
Handler: _StateStoreService_ListApplyStatus_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "PutPresence",
|
||||
Handler: _StateStoreService_PutPresence_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ListPresence",
|
||||
Handler: _StateStoreService_ListPresence_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ReadTaskLog",
|
||||
Handler: _StateStoreService_ReadTaskLog_Handler,
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Package presence keeps one record per running service instance in the State
|
||||
// Store, so the control plane can tell which services are up.
|
||||
package presence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/health"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
)
|
||||
|
||||
// Service names. Writer and reader share them so a record is never orphaned
|
||||
// by a spelling.
|
||||
const (
|
||||
Daemon = applystatus.Daemon
|
||||
Gateway = applystatus.Gateway
|
||||
Messaging = applystatus.Messaging
|
||||
UI = "archie-ui"
|
||||
StateStore = "archie-state-store"
|
||||
)
|
||||
|
||||
// Services is every service expected to report.
|
||||
func Services() []string { return []string{Daemon, Gateway, Messaging, UI, StateStore} }
|
||||
|
||||
// Stale reports whether a record's instance has stopped re-stamping it.
|
||||
func Stale(reportedAt, now time.Time) bool { return applystatus.Stale(reportedAt, now) }
|
||||
|
||||
// Build is what the binary was compiled as.
|
||||
type Build struct {
|
||||
Version string
|
||||
InstallType string
|
||||
}
|
||||
|
||||
// Run publishes this instance's record immediately and then on every restamp
|
||||
// interval until ctx ends. A nil registry reports ready with no probes.
|
||||
// Write failures are logged: the next tick retries.
|
||||
func Run(ctx context.Context, service string, build Build, store storecontract.PresenceStore, registry *health.Registry, log *slog.Logger) {
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
record := storecontract.Presence{
|
||||
Service: service, InstanceID: instanceID(), Version: build.Version,
|
||||
InstallType: build.InstallType, StartedAt: time.Now().UTC(),
|
||||
}
|
||||
ticker := time.NewTicker(applystatus.RestampInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
record.Ready, record.Detail = summarise(ctx, registry)
|
||||
record.ReportedAt = time.Now().UTC()
|
||||
if err := store.PutPresence(ctx, record); err != nil && ctx.Err() == nil {
|
||||
log.Warn("presence not reported", "service", service, "err", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func summarise(ctx context.Context, registry *health.Registry) (bool, string) {
|
||||
if registry == nil {
|
||||
return true, ""
|
||||
}
|
||||
report := registry.Run(ctx)
|
||||
var degraded []string
|
||||
for _, c := range report.Components {
|
||||
if !c.Ready {
|
||||
degraded = append(degraded, c.Name)
|
||||
}
|
||||
}
|
||||
return report.Status == health.StatusOK, strings.Join(degraded, ", ")
|
||||
}
|
||||
|
||||
func instanceID() string {
|
||||
b := make([]byte, 8)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
@@ -165,6 +165,13 @@ type ApplyStatusStore interface {
|
||||
ListApplyStatus(ctx context.Context) ([]ApplyStatus, error)
|
||||
}
|
||||
|
||||
// PresenceStore holds one record per running service instance. Writers are
|
||||
// the services themselves; readers are the UI and archied status.
|
||||
type PresenceStore interface {
|
||||
PutPresence(ctx context.Context, presence Presence) error
|
||||
ListPresence(ctx context.Context) ([]Presence, error)
|
||||
}
|
||||
|
||||
// MappingStore persists payload field mappings. Deliberately separate from
|
||||
// TaskStore and CaptureStore for the same reason those are split: the
|
||||
// dashboard's mapping editor should only acquire the mapping surface, not the
|
||||
@@ -323,6 +330,19 @@ type ApplyStatus struct {
|
||||
ReportedAt time.Time `json:"reported_at"`
|
||||
}
|
||||
|
||||
// Presence is one running service instance. Ready and Detail summarise its
|
||||
// readiness probes; Detail names the degraded ones.
|
||||
type Presence struct {
|
||||
Service string `json:"service"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
Version string `json:"version"`
|
||||
InstallType string `json:"install_type"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
ReportedAt time.Time `json:"reported_at"`
|
||||
Ready bool `json:"ready"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// WorkflowStat is one row of the per-workflow metrics table.
|
||||
type WorkflowStat struct {
|
||||
Workflow string `json:"workflow"`
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
-- One row per running service instance, re-stamped on the restamp interval.
|
||||
|
||||
-- +goose Up
|
||||
|
||||
CREATE TABLE presence (
|
||||
service text NOT NULL,
|
||||
instance_id text NOT NULL,
|
||||
version text NOT NULL,
|
||||
install_type text NOT NULL,
|
||||
started_at timestamptz NOT NULL,
|
||||
reported_at timestamptz NOT NULL,
|
||||
ready boolean NOT NULL,
|
||||
detail text NOT NULL DEFAULT '',
|
||||
PRIMARY KEY (service, instance_id)
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
|
||||
DROP TABLE presence;
|
||||
@@ -277,6 +277,17 @@ type PlaybookDispatch struct {
|
||||
DispatchedAt time.Time
|
||||
}
|
||||
|
||||
type Presence struct {
|
||||
Service string
|
||||
InstanceID string
|
||||
Version string
|
||||
InstallType string
|
||||
StartedAt time.Time
|
||||
ReportedAt time.Time
|
||||
Ready bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
type Resource struct {
|
||||
Kind string
|
||||
Value []byte
|
||||
|
||||
@@ -36,6 +36,15 @@ func (q *Queries) DeleteChannelStatusNotIn(ctx context.Context, dollar_1 []strin
|
||||
return err
|
||||
}
|
||||
|
||||
const deletePresenceBefore = `-- name: DeletePresenceBefore :exec
|
||||
DELETE FROM presence WHERE reported_at < $1
|
||||
`
|
||||
|
||||
func (q *Queries) DeletePresenceBefore(ctx context.Context, reportedAt time.Time) error {
|
||||
_, err := q.db.Exec(ctx, deletePresenceBefore, reportedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
const listApplyStatus = `-- name: ListApplyStatus :many
|
||||
SELECT process, kind, applied_version, error, reported_at
|
||||
FROM apply_status ORDER BY process, kind
|
||||
@@ -100,6 +109,40 @@ func (q *Queries) ListChannelStatus(ctx context.Context) ([]ChannelStatus, error
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listPresence = `-- name: ListPresence :many
|
||||
SELECT service, instance_id, version, install_type, started_at, reported_at, ready, detail
|
||||
FROM presence ORDER BY service, instance_id
|
||||
`
|
||||
|
||||
func (q *Queries) ListPresence(ctx context.Context) ([]Presence, error) {
|
||||
rows, err := q.db.Query(ctx, listPresence)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []Presence
|
||||
for rows.Next() {
|
||||
var i Presence
|
||||
if err := rows.Scan(
|
||||
&i.Service,
|
||||
&i.InstanceID,
|
||||
&i.Version,
|
||||
&i.InstallType,
|
||||
&i.StartedAt,
|
||||
&i.ReportedAt,
|
||||
&i.Ready,
|
||||
&i.Detail,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const upsertApplyStatus = `-- name: UpsertApplyStatus :exec
|
||||
INSERT INTO apply_status (process, kind, applied_version, error, reported_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
@@ -186,3 +229,40 @@ func (q *Queries) UpsertConfigSnapshot(ctx context.Context, arg UpsertConfigSnap
|
||||
_, err := q.db.Exec(ctx, upsertConfigSnapshot, arg.Schema, arg.Document, arg.PublishedAt)
|
||||
return err
|
||||
}
|
||||
|
||||
const upsertPresence = `-- name: UpsertPresence :exec
|
||||
INSERT INTO presence (service, instance_id, version, install_type, started_at, reported_at, ready, detail)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (service, instance_id) DO UPDATE SET
|
||||
version = excluded.version,
|
||||
install_type = excluded.install_type,
|
||||
started_at = excluded.started_at,
|
||||
reported_at = excluded.reported_at,
|
||||
ready = excluded.ready,
|
||||
detail = excluded.detail
|
||||
`
|
||||
|
||||
type UpsertPresenceParams struct {
|
||||
Service string
|
||||
InstanceID string
|
||||
Version string
|
||||
InstallType string
|
||||
StartedAt time.Time
|
||||
ReportedAt time.Time
|
||||
Ready bool
|
||||
Detail string
|
||||
}
|
||||
|
||||
func (q *Queries) UpsertPresence(ctx context.Context, arg UpsertPresenceParams) error {
|
||||
_, err := q.db.Exec(ctx, upsertPresence,
|
||||
arg.Service,
|
||||
arg.InstanceID,
|
||||
arg.Version,
|
||||
arg.InstallType,
|
||||
arg.StartedAt,
|
||||
arg.ReportedAt,
|
||||
arg.Ready,
|
||||
arg.Detail,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -42,3 +42,21 @@ ON CONFLICT (process, kind) DO UPDATE SET
|
||||
-- name: ListApplyStatus :many
|
||||
SELECT process, kind, applied_version, error, reported_at
|
||||
FROM apply_status ORDER BY process, kind;
|
||||
|
||||
-- name: UpsertPresence :exec
|
||||
INSERT INTO presence (service, instance_id, version, install_type, started_at, reported_at, ready, detail)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (service, instance_id) DO UPDATE SET
|
||||
version = excluded.version,
|
||||
install_type = excluded.install_type,
|
||||
started_at = excluded.started_at,
|
||||
reported_at = excluded.reported_at,
|
||||
ready = excluded.ready,
|
||||
detail = excluded.detail;
|
||||
|
||||
-- name: DeletePresenceBefore :exec
|
||||
DELETE FROM presence WHERE reported_at < $1;
|
||||
|
||||
-- name: ListPresence :many
|
||||
SELECT service, instance_id, version, install_type, started_at, reported_at, ready, detail
|
||||
FROM presence ORDER BY service, instance_id;
|
||||
@@ -123,3 +123,43 @@ func (s *Store) ListApplyStatus(ctx context.Context) ([]storecontract.ApplyStatu
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// PutPresence replaces one instance's record and drops records whose
|
||||
// instance stopped re-stamping long ago, so restarts do not accumulate rows.
|
||||
func (s *Store) PutPresence(ctx context.Context, presence storecontract.Presence) error {
|
||||
reported := presence.ReportedAt
|
||||
if reported.IsZero() {
|
||||
reported = time.Now()
|
||||
}
|
||||
q := s.queries()
|
||||
if err := q.UpsertPresence(ctx, postgresdb.UpsertPresenceParams{
|
||||
Service: presence.Service, InstanceID: presence.InstanceID, Version: presence.Version,
|
||||
InstallType: presence.InstallType, StartedAt: presence.StartedAt.UTC(), ReportedAt: reported.UTC(),
|
||||
Ready: presence.Ready, Detail: presence.Detail,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("store: put presence: %w", err)
|
||||
}
|
||||
if err := q.DeletePresenceBefore(ctx, reported.Add(-presenceRetention).UTC()); err != nil {
|
||||
return fmt.Errorf("store: prune presence: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// presenceRetention is how long a stopped instance stays listed as down.
|
||||
const presenceRetention = 24 * time.Hour
|
||||
|
||||
// ListPresence returns every instance's record, ordered by (service, instance).
|
||||
func (s *Store) ListPresence(ctx context.Context) ([]storecontract.Presence, error) {
|
||||
rows, err := s.queries().ListPresence(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store: list presence: %w", err)
|
||||
}
|
||||
out := make([]storecontract.Presence, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, storecontract.Presence{
|
||||
Service: r.Service, InstanceID: r.InstanceID, Version: r.Version, InstallType: r.InstallType,
|
||||
StartedAt: r.StartedAt, ReportedAt: r.ReportedAt, Ready: r.Ready, Detail: r.Detail,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -129,6 +129,7 @@ var (
|
||||
_ storecontract.ConfigSnapshotStore = (*Store)(nil)
|
||||
_ storecontract.ChannelStatusStore = (*Store)(nil)
|
||||
_ storecontract.ApplyStatusStore = (*Store)(nil)
|
||||
_ storecontract.PresenceStore = (*Store)(nil)
|
||||
_ task.Caller = (*Store)(nil)
|
||||
_ workflow.Store = (*Store)(nil)
|
||||
)
|
||||
|
||||
@@ -60,6 +60,7 @@ var (
|
||||
_ storecontract.ConfigSnapshotStore = (*Client)(nil)
|
||||
_ storecontract.ChannelStatusStore = (*Client)(nil)
|
||||
_ storecontract.ApplyStatusStore = (*Client)(nil)
|
||||
_ storecontract.PresenceStore = (*Client)(nil)
|
||||
_ storecontract.ExecutionCanceller = (*Client)(nil)
|
||||
_ identity.Repository = (*Client)(nil)
|
||||
_ storepkg.Manager = (*Client)(nil)
|
||||
@@ -431,6 +432,28 @@ func (c *Client) ListApplyStatus(ctx context.Context) ([]storecontract.ApplyStat
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
// PutPresence re-stamps this service instance's record. Administrative.
|
||||
func (c *Client) PutPresence(ctx context.Context, presence storecontract.Presence) error {
|
||||
_, err := c.client.PutPresence(ctx, &pb.PutPresenceRequest{Presence: presenceProto(presence)})
|
||||
if err != nil {
|
||||
return unmapError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListPresence reads every service instance's record.
|
||||
func (c *Client) ListPresence(ctx context.Context) ([]storecontract.Presence, error) {
|
||||
reply, err := c.client.ListPresence(ctx, &pb.ListPresenceRequest{})
|
||||
if err != nil {
|
||||
return nil, unmapError(err)
|
||||
}
|
||||
out := make([]storecontract.Presence, 0, len(reply.Presences))
|
||||
for _, presence := range reply.Presences {
|
||||
out = append(out, presenceValue(presence))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListCaptures streams one capture per message; see StreamCaptures in
|
||||
// server.go.
|
||||
func (c *Client) ListCaptures(ctx context.Context, limit int) ([]storecontract.CapturedEvent, error) {
|
||||
|
||||
@@ -36,6 +36,7 @@ var (
|
||||
errConfigSnapshotsUnavailable = status.Error(codes.Unavailable, "config snapshot store unavailable")
|
||||
errChannelStatusUnavailable = status.Error(codes.Unavailable, "channel status store unavailable")
|
||||
errApplyStatusUnavailable = status.Error(codes.Unavailable, "apply status store unavailable")
|
||||
errPresenceUnavailable = status.Error(codes.Unavailable, "presence store unavailable")
|
||||
errMappingUnavailable = status.Error(codes.Unavailable, "mapping store unavailable")
|
||||
errBindingUnavailable = status.Error(codes.Unavailable, "binding store unavailable")
|
||||
errSourceUnavailable = status.Error(codes.Unavailable, "source store unavailable")
|
||||
@@ -76,6 +77,7 @@ type Deps struct {
|
||||
// the honest answer for a store service no messaging process is writing to.
|
||||
ChannelStatus storecontract.ChannelStatusStore
|
||||
ApplyStatus storecontract.ApplyStatusStore
|
||||
Presence storecontract.PresenceStore
|
||||
Mappings storecontract.MappingStore
|
||||
EventTypes storecontract.EventTypeStore
|
||||
// MappingMatches counts the events each mapping resolved. Optional: nil
|
||||
@@ -556,6 +558,40 @@ func (s *server) GetConfigSnapshot(ctx context.Context, _ *pb.GetConfigSnapshotR
|
||||
return &pb.GetConfigSnapshotResponse{Snapshot: configSnapshotProto(snapshot), Found: true}, nil
|
||||
}
|
||||
|
||||
func (s *server) presence() (storecontract.PresenceStore, error) {
|
||||
if s.deps.Presence == nil {
|
||||
return nil, errPresenceUnavailable
|
||||
}
|
||||
return s.deps.Presence, nil
|
||||
}
|
||||
|
||||
func (s *server) PutPresence(ctx context.Context, r *pb.PutPresenceRequest) (*pb.PutPresenceResponse, error) {
|
||||
ps, err := s.presence()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ps.PutPresence(ctx, presenceValue(r.Presence)); err != nil {
|
||||
return nil, s.logErr("PutPresence", err)
|
||||
}
|
||||
return &pb.PutPresenceResponse{}, nil
|
||||
}
|
||||
|
||||
func (s *server) ListPresence(ctx context.Context, _ *pb.ListPresenceRequest) (*pb.ListPresenceResponse, error) {
|
||||
ps, err := s.presence()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
presences, err := ps.ListPresence(ctx)
|
||||
if err != nil {
|
||||
return nil, s.logErr("ListPresence", err)
|
||||
}
|
||||
out := make([]*pb.Presence, 0, len(presences))
|
||||
for _, presence := range presences {
|
||||
out = append(out, presenceProto(presence))
|
||||
}
|
||||
return &pb.ListPresenceResponse{Presences: out}, nil
|
||||
}
|
||||
|
||||
func (s *server) channelStatuses() (storecontract.ChannelStatusStore, error) {
|
||||
if s.deps.ChannelStatus == nil {
|
||||
return nil, errChannelStatusUnavailable
|
||||
|
||||
@@ -597,6 +597,35 @@ func applyStatusValue(status *pb.ApplyStatus) storecontract.ApplyStatus {
|
||||
}
|
||||
}
|
||||
|
||||
func presenceProto(presence storecontract.Presence) *pb.Presence {
|
||||
return &pb.Presence{
|
||||
Service: presence.Service,
|
||||
InstanceId: presence.InstanceID,
|
||||
Version: presence.Version,
|
||||
InstallType: presence.InstallType,
|
||||
StartedAt: timestamp(presence.StartedAt),
|
||||
ReportedAt: timestamp(presence.ReportedAt),
|
||||
Ready: presence.Ready,
|
||||
Detail: presence.Detail,
|
||||
}
|
||||
}
|
||||
|
||||
func presenceValue(presence *pb.Presence) storecontract.Presence {
|
||||
if presence == nil {
|
||||
return storecontract.Presence{}
|
||||
}
|
||||
return storecontract.Presence{
|
||||
Service: presence.Service,
|
||||
InstanceID: presence.InstanceId,
|
||||
Version: presence.Version,
|
||||
InstallType: presence.InstallType,
|
||||
StartedAt: timeValue(presence.StartedAt),
|
||||
ReportedAt: timeValue(presence.ReportedAt),
|
||||
Ready: presence.Ready,
|
||||
Detail: presence.Detail,
|
||||
}
|
||||
}
|
||||
|
||||
// taskLogEntryProto and taskLogEntryValue mirror internal/logging.Entry, whose
|
||||
// Fields map crosses as a JSON object string exactly as events.Event.Data does
|
||||
// (see eventDataJSON above). The logging package owns that format end to end;
|
||||
|
||||
@@ -120,6 +120,12 @@ service StateStoreService {
|
||||
rpc PutApplyStatus(PutApplyStatusRequest) returns (PutApplyStatusResponse);
|
||||
rpc ListApplyStatus(ListApplyStatusRequest) returns (ListApplyStatusResponse);
|
||||
|
||||
// Presence: every running service instance re-stamps one record, so a
|
||||
// record past the staleness window means that instance is down.
|
||||
// Administrative.
|
||||
rpc PutPresence(PutPresenceRequest) returns (PutPresenceResponse);
|
||||
rpc ListPresence(ListPresenceRequest) returns (ListPresenceResponse);
|
||||
|
||||
// Task log: one task attempt's persisted log. Task log files live in the
|
||||
// state directory this process owns, so the dashboard reads them over this
|
||||
// contract rather than opening them itself
|
||||
@@ -875,6 +881,23 @@ message PutApplyStatusResponse {}
|
||||
message ListApplyStatusRequest {}
|
||||
message ListApplyStatusResponse { repeated ApplyStatus statuses = 1; }
|
||||
|
||||
// Presence is one running service instance. ready and detail summarise its
|
||||
// readiness probes; detail names the degraded ones.
|
||||
message Presence {
|
||||
string service = 1;
|
||||
string instance_id = 2;
|
||||
string version = 3;
|
||||
string install_type = 4;
|
||||
google.protobuf.Timestamp started_at = 5;
|
||||
google.protobuf.Timestamp reported_at = 6;
|
||||
bool ready = 7;
|
||||
string detail = 8;
|
||||
}
|
||||
message PutPresenceRequest { Presence presence = 1; }
|
||||
message PutPresenceResponse {}
|
||||
message ListPresenceRequest {}
|
||||
message ListPresenceResponse { repeated Presence presences = 1; }
|
||||
|
||||
// TaskLogEntry mirrors internal/logging.Entry: one decoded log line. fields
|
||||
// carries every remaining key as a JSON object string, the same way Event
|
||||
// carries its data, so structured context survives the hop without this
|
||||
|
||||
Reference in new issue
Block a user