feat(controlplane): report which settings version each process is running
Every process that layers database-owned settings now publishes the version it applied, and re-stamps it every 30 seconds so the report time doubles as a liveness signal. A record older than 90 seconds belongs to a process that has stopped, which is the only way this surface can tell a live process from one that applied a version and then died. RuntimeConfig returned the merged config and discarded the versions it read, so no caller could report what it had applied. It now returns them, and archied and archie-gateway report each kind through boot.runtimeConfig at boot and on SIGHUP. A validation failure reports every kind with the error, because the layering is all-or-nothing, and leaves the recorded version alone so a rejected edit does not erase the fact that an older one is live. The daemon also reports workflow execution settings as the watch applies them, and archie-messaging reports channel settings at startup. Reporting is best-effort by construction: the configuration a report describes has already been applied, so a failed write is logged and never returned. That keeps it off the reload's error path, where it would have raised a banner saying a reload failed when its config took effect. applystatus.New refuses a process name Processes() does not list. The reader renders known names, so an invented one would be invisible rather than wrong, which is the same thing the page means by a process that never started. Relates to archie-core-pskb.
This commit is contained in:
1 parent
f38e1a30c8
commit
be4e29c91b
10 files changed
+401
-46
No files matched your search
@@ -28,6 +28,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/curator"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eda/module"
|
||||
"github.com/samcharles93/archie-core/internal/domain/eda/playbook"
|
||||
@@ -119,7 +120,15 @@ type boot struct {
|
||||
// plane published. They arrive on a watch rather than in the file
|
||||
// document, so reloadConfig re-applies them from here.
|
||||
executionSettings atomic.Pointer[workflow.ExecutionSettings]
|
||||
chatSessionStore gateway.SessionStore
|
||||
// applyStatus reports which control-plane resource version this process
|
||||
// is running. Nil until the State Store adapter is open, and nil-safe,
|
||||
// so apply points call it without a guard.
|
||||
applyStatus *applystatus.Reporter
|
||||
// processName is which binary this composition is running as, and is the
|
||||
// name apply-status records carry. Set by each entry point, because one
|
||||
// boot builds both archied and archie-gateway.
|
||||
processName string
|
||||
chatSessionStore gateway.SessionStore
|
||||
|
||||
catalog modelcatalog.Snapshot
|
||||
catalogModels []string
|
||||
@@ -349,6 +358,7 @@ func (b *boot) openStateStoreAdapter() error {
|
||||
}
|
||||
b.stateStore = client
|
||||
b.controlPlane = controlplane.NewRPCClient(client.ControlPlane())
|
||||
b.applyStatus = applystatus.New(b.processName, client, b.log)
|
||||
b.stateStoreGrants = &staterpc.GrantIssuer{Client: client}
|
||||
b.stateStoreToken = b.cfg.Services.ResolvedToken(config.ServiceNameState, b.secrets.Getenv)
|
||||
b.addCleanup(cleanup)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/configuration"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
// as it stands reverts each of these layers to its file value until the
|
||||
// process restarts (archie-core-ju85).
|
||||
func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Config, error) {
|
||||
cfg, err := b.controlPlane.RuntimeConfig(ctx, base)
|
||||
cfg, versions, err := b.controlPlane.RuntimeConfig(ctx, base)
|
||||
if err != nil {
|
||||
return config.Config{}, err
|
||||
}
|
||||
@@ -24,11 +25,23 @@ func (b *boot) runtimeConfig(ctx context.Context, base config.Config) (config.Co
|
||||
applyExecutionBudgets(&cfg, *settings)
|
||||
}
|
||||
if err := configuration.Validate(&cfg); err != nil {
|
||||
return config.Config{}, fmt.Errorf("validate database settings: %w", err)
|
||||
wrapped := fmt.Errorf("validate database settings: %w", err)
|
||||
b.reportApplied(ctx, versions, wrapped)
|
||||
return config.Config{}, wrapped
|
||||
}
|
||||
b.reportApplied(ctx, versions, nil)
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// reportApplied publishes the version of each kind this process just layered
|
||||
// in. On a validation failure every kind is reported with the error, because
|
||||
// the layering is all-or-nothing: none of the versions read took effect.
|
||||
func (b *boot) reportApplied(ctx context.Context, versions map[string]int64, applyErr error) {
|
||||
for kind, version := range versions {
|
||||
b.applyStatus.Report(ctx, kind, version, applyErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *boot) loadRuntimeConfig(ctx context.Context) error {
|
||||
cfg, err := b.runtimeConfig(ctx, b.cfg)
|
||||
if err != nil {
|
||||
@@ -77,7 +90,7 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(settings, version)
|
||||
b.applyWorkflowExecutionSettings(ctx, settings, version)
|
||||
updates, err := b.controlPlane.WatchWorkflowExecutionSettings(ctx, version)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -88,7 +101,7 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
b.log.Error("workflow execution settings watch failed", "err", update.Err)
|
||||
return
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(update.Settings, update.Version)
|
||||
b.applyWorkflowExecutionSettings(ctx, update.Settings, update.Version)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
@@ -97,11 +110,12 @@ func (b *boot) startWorkflowExecutionSettings(ctx context.Context) error {
|
||||
// applyWorkflowExecutionSettings records the settings as well as applying
|
||||
// them: they arrive on a watch rather than in the file document, so a reload
|
||||
// has nowhere else to read them back from.
|
||||
func (b *boot) applyWorkflowExecutionSettings(settings workflow.ExecutionSettings, version int64) {
|
||||
func (b *boot) applyWorkflowExecutionSettings(ctx context.Context, settings workflow.ExecutionSettings, version int64) {
|
||||
b.executionSettings.Store(&settings)
|
||||
cfg := b.cfgHolder.Get().Clone()
|
||||
applyExecutionBudgets(&cfg, settings)
|
||||
b.cfgHolder.Set(cfg)
|
||||
b.applyStatus.Report(ctx, controlplane.WorkflowExecutionSettingsKind, version, nil)
|
||||
b.log.Info("workflow execution settings applied", "version", version)
|
||||
}
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ func TestReloadConfigKeepsDatabaseOwnedSettings(t *testing.T) {
|
||||
if err := b.loadRuntimeConfig(t.Context()); err != nil {
|
||||
t.Fatalf("loadRuntimeConfig: %v", err)
|
||||
}
|
||||
b.applyWorkflowExecutionSettings(workflow.ExecutionSettings{
|
||||
b.applyWorkflowExecutionSettings(t.Context(), workflow.ExecutionSettings{
|
||||
MaxModelToolSteps: 99, MaxRuntime: time.Hour, MaxConsecutiveGateFailures: 5,
|
||||
}, 4)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/events"
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
|
||||
@@ -55,6 +56,7 @@ func gatewayListenAndToken(b *boot, options GatewayOptions) (listen, token strin
|
||||
// step 8), which is out of state-store scope.
|
||||
func RunGateway(ctx context.Context, options GatewayOptions) error {
|
||||
b := newBootstrap()
|
||||
b.processName = applystatus.Gateway
|
||||
defer b.cleanup()
|
||||
if err := b.loadConfig(ctx, options.Config, options.Overlay); err != nil {
|
||||
return err
|
||||
@@ -132,6 +134,7 @@ func (b *boot) openGatewayState(ctx context.Context) error {
|
||||
if err := b.loadRuntimeConfig(ctx); err != nil {
|
||||
return fmt.Errorf("load runtime settings: %w", err)
|
||||
}
|
||||
go b.applyStatus.Run(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/container"
|
||||
"github.com/samcharles93/archie-core/internal/daemon"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/curator"
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
"github.com/samcharles93/archie-core/internal/domain/workflow"
|
||||
@@ -345,6 +346,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
defer stop()
|
||||
|
||||
b := newBootstrap()
|
||||
b.processName = applystatus.Daemon
|
||||
// Arm the shutdown watchdog before any cleanup is registered so its
|
||||
// disarm runs last in the LIFO cleanup chain: it force-exits the process
|
||||
// as a backstop if graceful shutdown hangs past the drain-plus-grace
|
||||
@@ -385,6 +387,7 @@ func Run() int { //nolint:cyclop,funlen // the composition root's setup sequence
|
||||
b.log.Error("workflow execution settings unavailable", "err", err)
|
||||
return 1
|
||||
}
|
||||
go b.applyStatus.Run(ctx)
|
||||
if exit, err := b.handleRequeue(ctx, args.requeue, args.once); err != nil {
|
||||
return 1
|
||||
} else if exit {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/app/controlplane"
|
||||
"github.com/samcharles93/archie-core/internal/config"
|
||||
"github.com/samcharles93/archie-core/internal/domain/applystatus"
|
||||
"github.com/samcharles93/archie-core/internal/domain/messaging"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/gatewayrpc"
|
||||
"github.com/samcharles93/archie-core/internal/infrastructure/staterpc"
|
||||
@@ -38,12 +39,15 @@ func Run(ctx context.Context, o Options) error {
|
||||
}
|
||||
closeStateStore = closeClient
|
||||
defer closeStateStore()
|
||||
chatSettings, settingsErr := controlplane.NewRPCClient(stateStore.ControlPlane()).RuntimeChatConfig(ctx, config.ChatConfig{
|
||||
chatSettings, channelVersion, settingsErr := controlplane.NewRPCClient(stateStore.ControlPlane()).RuntimeChatConfig(ctx, config.ChatConfig{
|
||||
Telegram: cfg.Telegram, Email: cfg.Email, Webhook: cfg.Webhook, WebhookAddr: cfg.WebhookAddr, ShowToolCalls: cfg.ShowToolCalls,
|
||||
})
|
||||
if settingsErr != nil {
|
||||
return fmt.Errorf("load channel settings: %w", settingsErr)
|
||||
}
|
||||
reporter := applystatus.New(applystatus.Messaging, stateStore, log)
|
||||
reporter.Report(ctx, controlplane.ChannelSettingsKind, channelVersion, nil)
|
||||
go reporter.Run(ctx)
|
||||
secrets := secret.NewRegistry()
|
||||
cfg.Telegram, cfg.Email, cfg.Webhook, cfg.WebhookAddr, cfg.ShowToolCalls = chatSettings.Telegram, chatSettings.Email, chatSettings.Webhook, chatSettings.WebhookAddr, chatSettings.ShowToolCalls
|
||||
cfg.TelegramToken, settingsErr = resolveTelegramToken(chatSettings.Telegram, secrets)
|
||||
|
||||
@@ -14,9 +14,14 @@ import (
|
||||
// configuration. Values intentionally absent from a control-plane projection,
|
||||
// such as secret-bearing MCP headers and channel update commands, remain
|
||||
// bootstrap-owned.
|
||||
func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, error) {
|
||||
//
|
||||
// It also returns the version of each kind it read, so the process that
|
||||
// layers them in can report which version it is running
|
||||
// (docs/prds/control-plane-apply-status.md).
|
||||
func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.Config, map[string]int64, error) {
|
||||
out := base.Clone()
|
||||
if err := c.query(ctx, ProviderSettingsKind, func(value []byte) error {
|
||||
versions := map[string]int64{}
|
||||
if err := c.queryInto(ctx, versions, ProviderSettingsKind, func(value []byte) error {
|
||||
var providers map[string]providerDocument
|
||||
if err := json.Unmarshal(value, &providers); err != nil {
|
||||
return err
|
||||
@@ -27,20 +32,20 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return config.Config{}, err
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSON(ctx, ModelRoleAssignmentsKind, &out.Models); err != nil {
|
||||
return config.Config{}, err
|
||||
if err := c.queryJSONInto(ctx, versions, ModelRoleAssignmentsKind, &out.Models); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSON(ctx, RepositoryPoliciesKind, &out.Repos); err != nil {
|
||||
return config.Config{}, err
|
||||
if err := c.queryJSONInto(ctx, versions, RepositoryPoliciesKind, &out.Repos); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
chat, err := c.RuntimeChatConfig(ctx, out.Chat)
|
||||
chat, chatVersion, err := c.RuntimeChatConfig(ctx, out.Chat)
|
||||
if err != nil {
|
||||
return config.Config{}, err
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
out.Chat = chat
|
||||
if err := c.query(ctx, SchedulingPolicyKind, func(value []byte) error {
|
||||
out.Chat, versions[ChannelSettingsKind] = chat, chatVersion
|
||||
if err := c.queryInto(ctx, versions, SchedulingPolicyKind, func(value []byte) error {
|
||||
var policy schedulingPolicy
|
||||
if err := json.Unmarshal(value, &policy); err != nil {
|
||||
return err
|
||||
@@ -52,13 +57,13 @@ func (c *Client) RuntimeConfig(ctx context.Context, base config.Config) (config.
|
||||
out.PollInterval, out.MaxRetries, out.Dispatch = config.Duration(interval), policy.MaxRetries, policy.Dispatch
|
||||
return nil
|
||||
}); err != nil {
|
||||
return config.Config{}, err
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
return c.runtimeToolConfig(ctx, out)
|
||||
return c.runtimeToolConfig(ctx, versions, out)
|
||||
}
|
||||
|
||||
func (c *Client) runtimeToolConfig(ctx context.Context, out config.Config) (config.Config, error) {
|
||||
if err := c.query(ctx, ToolSettingsKind, func(value []byte) error {
|
||||
func (c *Client) runtimeToolConfig(ctx context.Context, versions map[string]int64, out config.Config) (config.Config, map[string]int64, error) {
|
||||
if err := c.queryInto(ctx, versions, ToolSettingsKind, func(value []byte) error {
|
||||
var settings toolSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -74,9 +79,9 @@ func (c *Client) runtimeToolConfig(ctx context.Context, out config.Config) (conf
|
||||
out.Tools = config.ToolsConfig{MCPServers: servers, Policy: settings.Policy, WebFetch: settings.WebFetch, Minimax: config.MinimaxConfig{Enabled: settings.Minimax.Enabled, APIKey: settings.Minimax.APIKey, BaseURL: settings.Minimax.BaseURL}}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return config.Config{}, err
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.query(ctx, PluginSettingsKind, func(value []byte) error {
|
||||
if err := c.queryInto(ctx, versions, PluginSettingsKind, func(value []byte) error {
|
||||
var settings pluginSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -84,17 +89,17 @@ func (c *Client) runtimeToolConfig(ctx context.Context, out config.Config) (conf
|
||||
out.PluginDir, out.ModuleDir, out.SecretEngineDir, out.SkillsDir = settings.PluginDir, settings.ModuleDir, settings.SecretEngineDir, settings.SkillsDir
|
||||
return nil
|
||||
}); err != nil {
|
||||
return config.Config{}, err
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
if err := c.queryJSON(ctx, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
|
||||
return config.Config{}, err
|
||||
if err := c.queryJSONInto(ctx, versions, ContainerRuntimePoliciesKind, &out.Containers); err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
return out, nil
|
||||
return out, versions, nil
|
||||
}
|
||||
|
||||
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, error) {
|
||||
func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig) (config.ChatConfig, int64, error) {
|
||||
out := base
|
||||
err := c.query(ctx, ChannelSettingsKind, func(value []byte) error {
|
||||
version, err := c.query(ctx, ChannelSettingsKind, func(value []byte) error {
|
||||
var settings channelSettings
|
||||
if err := json.Unmarshal(value, &settings); err != nil {
|
||||
return err
|
||||
@@ -109,23 +114,36 @@ func (c *Client) RuntimeChatConfig(ctx context.Context, base config.ChatConfig)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
return out, version, err
|
||||
}
|
||||
|
||||
func (c *Client) queryJSON(ctx context.Context, kind string, target any) error {
|
||||
return c.query(ctx, kind, func(value []byte) error { return json.Unmarshal(value, target) })
|
||||
}
|
||||
|
||||
func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) error {
|
||||
response, err := c.rpc.Query(ctx, &pb.QueryRequest{Kind: kind})
|
||||
// queryInto decodes a resource and records the version it came from, so
|
||||
// RuntimeConfig ends up holding the version of every kind it layered in.
|
||||
func (c *Client) queryInto(ctx context.Context, versions map[string]int64, kind string, decode func([]byte) error) error {
|
||||
version, err := c.query(ctx, kind, decode)
|
||||
if err != nil {
|
||||
return clientError(err)
|
||||
}
|
||||
if response.Resource == nil {
|
||||
return fmt.Errorf("%s resource missing", kind)
|
||||
}
|
||||
if err := decode(response.Resource.ValueJson); err != nil {
|
||||
return fmt.Errorf("decode %s: %w", kind, err)
|
||||
return err
|
||||
}
|
||||
versions[kind] = version
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) queryJSONInto(ctx context.Context, versions map[string]int64, kind string, target any) error {
|
||||
return c.queryInto(ctx, versions, kind, func(value []byte) error { return json.Unmarshal(value, target) })
|
||||
}
|
||||
|
||||
// query returns the version of the resource it decoded, so callers that layer
|
||||
// a resource in can report the version they applied.
|
||||
func (c *Client) query(ctx context.Context, kind string, decode func([]byte) error) (int64, error) {
|
||||
response, err := c.rpc.Query(ctx, &pb.QueryRequest{Kind: kind})
|
||||
if err != nil {
|
||||
return 0, clientError(err)
|
||||
}
|
||||
if response.Resource == nil {
|
||||
return 0, fmt.Errorf("%s resource missing", kind)
|
||||
}
|
||||
if err := decode(response.Resource.ValueJson); err != nil {
|
||||
return 0, fmt.Errorf("decode %s: %w", kind, err)
|
||||
}
|
||||
return response.Resource.Version, nil
|
||||
}
|
||||
@@ -55,10 +55,20 @@ func TestRuntimeConfigUsesDatabaseResourcesAndPreservesBootstrapOnlySecrets(t *t
|
||||
ContainerRuntimePoliciesKind: config.ContainerConfig{LegacyEnabled: true, Image: "archie:next", PullPolicy: "missing"},
|
||||
}})
|
||||
|
||||
got, err := client.RuntimeConfig(t.Context(), base)
|
||||
got, versions, err := client.RuntimeConfig(t.Context(), base)
|
||||
if err != nil {
|
||||
t.Fatalf("RuntimeConfig: %v", err)
|
||||
}
|
||||
// Every kind it read is reported as applied, so the version a process
|
||||
// publishes is the one it actually layered in (archie-core-pskb).
|
||||
for _, kind := range []string{
|
||||
ProviderSettingsKind, ModelRoleAssignmentsKind, RepositoryPoliciesKind, ChannelSettingsKind,
|
||||
SchedulingPolicyKind, ToolSettingsKind, PluginSettingsKind, ContainerRuntimePoliciesKind,
|
||||
} {
|
||||
if versions[kind] != 2 {
|
||||
t.Errorf("versions[%s] = %d, want the 2 the store answered with", kind, versions[kind])
|
||||
}
|
||||
}
|
||||
if got.Models["builder"] != "main/model" || got.Repos[0].FullName() != "acme/widget" || got.MaxRetries != 7 || got.PluginDir != "/plugins" || got.Containers.Image != "archie:next" {
|
||||
t.Fatalf("database settings not applied: %+v", got)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
// Package applystatus reports which version of a control-plane resource each
|
||||
// process is running. See docs/prds/control-plane-apply-status.md.
|
||||
package applystatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
)
|
||||
|
||||
// The processes that apply control-plane resources. Writer and reader share
|
||||
// these names: a process reporting under a name the reader does not know is
|
||||
// indistinguishable from one that never reported, so both sides take them
|
||||
// from here.
|
||||
const (
|
||||
Daemon = "archied"
|
||||
Gateway = "archie-gateway"
|
||||
Messaging = "archie-messaging"
|
||||
)
|
||||
|
||||
// Processes is every name a record may carry, for the reader to render and
|
||||
// for the test that binds writer to reader.
|
||||
func Processes() []string { return []string{Daemon, Gateway, Messaging} }
|
||||
|
||||
const (
|
||||
// RestampInterval is how often a process rewrites its records. It is the
|
||||
// only thing that distinguishes a live process from one that applied a
|
||||
// version and then died.
|
||||
RestampInterval = 30 * time.Second
|
||||
// StaleAfter is how old a record may be before its process is treated as
|
||||
// gone. Three intervals, so a single missed re-stamp is not a death.
|
||||
StaleAfter = 3 * RestampInterval
|
||||
)
|
||||
|
||||
// Stale reports whether a record's process has stopped re-stamping it.
|
||||
func Stale(reportedAt, now time.Time) bool {
|
||||
return now.Sub(reportedAt) > StaleAfter
|
||||
}
|
||||
|
||||
// Reporter publishes one process's apply status and keeps it re-stamped.
|
||||
// A nil Reporter reports nothing, so a process with no State Store wiring
|
||||
// calls it without a guard at each site.
|
||||
type Reporter struct {
|
||||
process string
|
||||
store storecontract.ApplyStatusStore
|
||||
log *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
applied map[string]storecontract.ApplyStatus
|
||||
}
|
||||
|
||||
// New returns a Reporter for one process, or nil when there is no store to
|
||||
// report to or the composition is a process that applies nothing, such as the
|
||||
// State Store itself.
|
||||
//
|
||||
// A name Processes() does not list is refused rather than reported: the reader
|
||||
// renders the names it knows, so an invented one would vanish from the page
|
||||
// exactly as a process that never started does. Failing here makes a new
|
||||
// process declare itself in one place.
|
||||
func New(process string, store storecontract.ApplyStatusStore, log *slog.Logger) *Reporter {
|
||||
if store == nil || process == "" {
|
||||
return nil
|
||||
}
|
||||
if log == nil {
|
||||
log = slog.New(slog.DiscardHandler)
|
||||
}
|
||||
if !slices.Contains(Processes(), process) {
|
||||
log.Error("apply status disabled: unknown process name", "process", process, "known", Processes())
|
||||
return nil
|
||||
}
|
||||
return &Reporter{
|
||||
process: process, store: store, log: log, now: time.Now,
|
||||
applied: map[string]storecontract.ApplyStatus{},
|
||||
}
|
||||
}
|
||||
|
||||
// Report records that this process applied version of kind, or failed to.
|
||||
// A failure leaves the recorded version alone: an edit this process rejected
|
||||
// does not change which version is live in it.
|
||||
//
|
||||
// Best-effort by construction. The configuration a report describes has
|
||||
// already been applied, so a failed report is logged and never returned.
|
||||
func (r *Reporter) Report(ctx context.Context, kind string, version int64, applyErr error) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
status := storecontract.ApplyStatus{
|
||||
Process: r.process, Kind: kind, AppliedVersion: version, ReportedAt: r.now().UTC(),
|
||||
}
|
||||
if applyErr != nil {
|
||||
status.Error = applyErr.Error()
|
||||
r.mu.Lock()
|
||||
if previous, ok := r.applied[kind]; ok {
|
||||
status.AppliedVersion = previous.AppliedVersion
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.applied[kind] = status
|
||||
r.mu.Unlock()
|
||||
r.write(ctx, status)
|
||||
}
|
||||
|
||||
// Run re-stamps this process's records until ctx ends.
|
||||
func (r *Reporter) Run(ctx context.Context) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(RestampInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.restamp(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reporter) restamp(ctx context.Context) {
|
||||
r.mu.Lock()
|
||||
kinds := slices.Sorted(maps.Keys(r.applied))
|
||||
now := r.now().UTC()
|
||||
statuses := make([]storecontract.ApplyStatus, 0, len(kinds))
|
||||
for _, kind := range kinds {
|
||||
status := r.applied[kind]
|
||||
status.ReportedAt = now
|
||||
r.applied[kind] = status
|
||||
statuses = append(statuses, status)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
for _, status := range statuses {
|
||||
r.write(ctx, status)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reporter) write(ctx context.Context, status storecontract.ApplyStatus) {
|
||||
if err := r.store.PutApplyStatus(ctx, status); err != nil {
|
||||
r.log.Warn("apply status not reported", "kind", status.Kind, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package applystatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
||||
)
|
||||
|
||||
type recordingStore struct {
|
||||
mu sync.Mutex
|
||||
writes []storecontract.ApplyStatus
|
||||
failNext error
|
||||
}
|
||||
|
||||
func (s *recordingStore) PutApplyStatus(_ context.Context, status storecontract.ApplyStatus) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.failNext != nil {
|
||||
err := s.failNext
|
||||
s.failNext = nil
|
||||
return err
|
||||
}
|
||||
s.writes = append(s.writes, status)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *recordingStore) ListApplyStatus(context.Context) ([]storecontract.ApplyStatus, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]storecontract.ApplyStatus(nil), s.writes...), nil
|
||||
}
|
||||
|
||||
func (s *recordingStore) snapshot() []storecontract.ApplyStatus {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]storecontract.ApplyStatus(nil), s.writes...)
|
||||
}
|
||||
|
||||
func TestReportRecordsVersionAndError(t *testing.T) {
|
||||
store := &recordingStore{}
|
||||
at := time.Date(2026, 9, 21, 10, 0, 0, 0, time.UTC)
|
||||
r := New(Daemon, store, nil)
|
||||
r.now = func() time.Time { return at }
|
||||
|
||||
r.Report(t.Context(), "tool-settings", 4, nil)
|
||||
r.Report(t.Context(), "model-role-assignments", 2, errors.New("validate database settings: bad role"))
|
||||
|
||||
got := store.snapshot()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("wrote %d records, want 2", len(got))
|
||||
}
|
||||
if got[0].Process != Daemon || got[0].Kind != "tool-settings" || got[0].AppliedVersion != 4 || got[0].Error != "" {
|
||||
t.Errorf("first record = %+v, want a clean apply of version 4 by %s", got[0], Daemon)
|
||||
}
|
||||
if got[1].Error == "" || got[1].AppliedVersion != 2 {
|
||||
t.Errorf("second record = %+v, want the error alongside the version still live", got[1])
|
||||
}
|
||||
if !got[0].ReportedAt.Equal(at) {
|
||||
t.Errorf("ReportedAt = %v, want %v", got[0].ReportedAt, at)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportIsBestEffort pins that a failed write never reaches the caller: a
|
||||
// report describes configuration that has already been applied, and failing
|
||||
// the apply because the report failed would be the wrong way round.
|
||||
func TestReportIsBestEffort(t *testing.T) {
|
||||
store := &recordingStore{failNext: errors.New("state store unreachable")}
|
||||
New(Daemon, store, nil).Report(t.Context(), "tool-settings", 1, nil)
|
||||
}
|
||||
|
||||
// TestRestampRewritesEveryKnownKind pins the liveness signal: the report time
|
||||
// has to keep moving while the process lives, because the UI reads a record
|
||||
// that stopped moving as a process that stopped running.
|
||||
func TestRestampRewritesEveryKnownKind(t *testing.T) {
|
||||
store := &recordingStore{}
|
||||
at := time.Date(2026, 9, 21, 10, 0, 0, 0, time.UTC)
|
||||
r := New(Daemon, store, nil)
|
||||
r.now = func() time.Time { return at }
|
||||
|
||||
r.Report(t.Context(), "tool-settings", 4, nil)
|
||||
r.Report(t.Context(), "plugin-settings", 9, nil)
|
||||
|
||||
at = at.Add(RestampInterval)
|
||||
r.restamp(t.Context())
|
||||
|
||||
got := store.snapshot()
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("wrote %d records, want the 2 reports plus 2 re-stamps", len(got))
|
||||
}
|
||||
versions := map[string]int64{"tool-settings": 4, "plugin-settings": 9}
|
||||
for _, record := range got[2:] {
|
||||
if !record.ReportedAt.Equal(at) {
|
||||
t.Errorf("re-stamped %s at %v, want %v", record.Kind, record.ReportedAt, at)
|
||||
}
|
||||
if record.AppliedVersion != versions[record.Kind] {
|
||||
t.Errorf("re-stamp changed %s from %d to %d", record.Kind, versions[record.Kind], record.AppliedVersion)
|
||||
}
|
||||
delete(versions, record.Kind)
|
||||
}
|
||||
if len(versions) != 0 {
|
||||
t.Errorf("re-stamp missed %v", versions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStale(t *testing.T) {
|
||||
at := time.Date(2026, 9, 21, 10, 0, 0, 0, time.UTC)
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
age time.Duration
|
||||
want bool
|
||||
}{
|
||||
{"just reported", 0, false},
|
||||
{"one interval", RestampInterval, false},
|
||||
{"inside the window", StaleAfter - time.Second, false},
|
||||
{"past the window", StaleAfter + time.Second, true},
|
||||
} {
|
||||
if got := Stale(at, at.Add(tt.age)); got != tt.want {
|
||||
t.Errorf("%s: Stale = %v, want %v", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewRefusesAnUnknownProcessName binds writer to reader: the reader
|
||||
// renders the names Processes() lists, so a process reporting under any other
|
||||
// name would be invisible rather than wrong. It must not report at all.
|
||||
func TestNewRefusesAnUnknownProcessName(t *testing.T) {
|
||||
store := &recordingStore{}
|
||||
r := New("archie-daemon", store, nil)
|
||||
if r != nil {
|
||||
t.Fatal("New accepted a process name Processes() does not list")
|
||||
}
|
||||
r.Report(t.Context(), "tool-settings", 1, nil)
|
||||
if len(store.snapshot()) != 0 {
|
||||
t.Fatal("a refused reporter still wrote a record")
|
||||
}
|
||||
for _, process := range Processes() {
|
||||
if New(process, store, nil) == nil {
|
||||
t.Errorf("New(%q) = nil, want a reporter for a listed process", process)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user