diff --git a/docs/prds/status-health-surface.md b/docs/prds/status-health-surface.md index 3dc673649..a1c9665a4 100644 --- a/docs/prds/status-health-surface.md +++ b/docs/prds/status-health-surface.md @@ -3,9 +3,10 @@ **Status:** Finalised **Beads issue:** `archie-core-wp9s` -A reported field must have a truthful source. Broker connectivity, container -pool, last poll, provider reachability and channel state are reported; a field -with no truthful source is left out rather than faked. +A reported field must have a truthful source. `/status` reports broker +connectivity and the last chat-model call; the container pool, last poll and +channel state have no source in the process that answers the command, so they +are left out rather than faked. ## Background @@ -40,7 +41,9 @@ something to resolve by implication here. Each bullet below was checked against real code before being called cheap or not; nothing here is guessed. Both open questions were settled (see "The two -decisions") and all five fields shipped. +decisions"). The health section reports the broker connection and the last +chat-model call; the three facts the answering process cannot see are covered +under "Truthful or absent". ### Verified as cheap against the tree @@ -93,10 +96,13 @@ decisions") and all five fields shipped. - **Queue depth.** Covered by Phase 1. - **Rendering.** `gateway.formatHealth` renders one line per reported fact (`Broker:`, `Containers:`, `Channels:`, `Chat model:`, `Last poll:`) between - the queue line and the runtime block. The sources it reads are functions, not - captured values: occupancy and channel state change over the process's life, - and the daemon is built after the gateways, so a source captured at - construction would freeze `/status` on boot-time state. + the queue line and the runtime block. A `/status` reply carries `Broker:` and + `Chat model:`: the Gateway is the only process that serves a Router, and it + holds neither a container pool, a poll loop nor a channel manager. The other + three lines render when a report carries them. The broker source is a + function rather than a captured value because the connection differs by + composition: the daemon's shared eventbus client in one process, the + Gateway's own task-actions connection in the other. ### Truthful or absent -- what is deliberately not reported @@ -114,7 +120,9 @@ decisions") and all five fields shipped. answering the command. **Decided: they stay per-process.** Pooling them would mean a new daemon↔gateway health RPC for one status line, and a per-process line is true where an aggregate would be a claim about a process - the answering one cannot see. + the answering one cannot see. The composition root therefore builds the + health source from the broker connection and the chat-model recorder alone: a + producer for a daemon-owned fact would build a section no reply can carry. - **A channel's failure reason can outlive the failure.** `status.Manager` keeps the last non-empty `Detail` across state changes, and that one field carries both a descriptor's standing caveat and a runtime failure reason, so a channel diff --git a/internal/app/archied/bootstrap.go b/internal/app/archied/bootstrap.go index bdf3f02a0..09c7fa870 100644 --- a/internal/app/archied/bootstrap.go +++ b/internal/app/archied/bootstrap.go @@ -161,8 +161,9 @@ type boot struct { // this process makes, read back by /status (newStatusHealth). Built before // the chat runtime, which carries it into each turn runner. providerOutcomes *providerOutcomeRecorder - // statusHealth is the /status health source for this process, built from - // the subsystems that exist here. + // statusHealth is the /status health source this process serves. It + // carries the broker connection and the chat-model outcomes; see + // newStatusHealth for the facts it deliberately leaves out. statusHealth gateway.HealthSource // rateLimiter is the shared per-(channel, sender) inbound budget every // chat Router is given. Nil when [chat.rate_limit] is not configured, diff --git a/internal/app/archied/gateway.go b/internal/app/archied/gateway.go index 7be7b2527..5363e5107 100644 --- a/internal/app/archied/gateway.go +++ b/internal/app/archied/gateway.go @@ -101,9 +101,8 @@ func RunGateway(ctx context.Context, options GatewayOptions) error { return fmt.Errorf("connect gateway task actions: %w", err) } b.addCleanup(nc.Close) - // Held before the chat runtime is built so /status' health source can - // report this process's own broker connection (newStatusHealth reads it - // lazily, but the field must be set before requests can arrive). + // Held so /status can report this process's own broker connection: the + // health source reads it through boot rather than dialling. b.taskActionsConn = nc contract, err := b.startGatewayRuntime(ctx, taskactions.Client{Conn: nc, Timeout: 30 * time.Second}) if err != nil { diff --git a/internal/app/archied/gateway_runtime.go b/internal/app/archied/gateway_runtime.go index d72b3e504..712daad17 100644 --- a/internal/app/archied/gateway_runtime.go +++ b/internal/app/archied/gateway_runtime.go @@ -52,8 +52,7 @@ func (b *boot) setupChatRuntime(ctx context.Context, cfg config.Config) error { // ── Operator health surface ────────────────────────────────────── // Built before the gateways because every turn runner and router built // later carries both: the recorder is written by sendChatTurn, the source - // is read by /status. See newStatusHealth for why the source reads boot's - // fields lazily. + // is read by /status. b.providerOutcomes = newProviderOutcomeRecorder() b.statusHealth = newStatusHealth(b) diff --git a/internal/app/archied/status_health.go b/internal/app/archied/status_health.go index 74efec41a..a4d9ad148 100644 --- a/internal/app/archied/status_health.go +++ b/internal/app/archied/status_health.go @@ -1,21 +1,18 @@ // status_health.go is the composition-root bridge behind /status' health -// section: it reads whichever subsystems THIS process owns and hands /status a -// narrow report, so the gateway package never takes a daemon, pool or manager -// handle. +// section: it reads the facts THIS process owns and hands /status a narrow +// report, so the gateway package never takes a daemon, pool or manager handle. // -// The split matters because archie's operator surface spans two processes. -// The daemon owns the forge poll loop, the container pool and the channel -// manager; the standalone Gateway owns none of those. Each process therefore -// reports what it actually has and leaves the rest absent -- /status shows a -// shorter list in the Gateway, not a list of zeroes. +// The report carries a broker line and a chat-model line. Those are the facts +// the process that serves /status holds: setupGatewayChat is the sole +// production constructor of a Router and it runs in the Gateway, which owns +// neither a container pool, a poll loop nor a channel manager. A producer +// wired for one of the three would build a section no surface ever reads. package archied import ( - "cmp" "sync" "time" - "github.com/samcharles93/archie-core/internal/channels/status" "github.com/samcharles93/archie-core/internal/gateway" ) @@ -83,18 +80,12 @@ func (r *providerOutcomeRecorder) LastChatModelOutcome() (gateway.ChatModelOutco // subsystems. Every source is optional; a nil source means this process holds // no truthful input for that fact and the section is left out of the report. // -// The sources are functions rather than captured values because they change -// over the life of the process: the container pool's occupancy, the channel -// states, and the poll timestamp all move, and the two NATS connections can -// drop. Capturing them at construction would freeze /status on the daemon's -// boot-time state -- occupancy would read 0/4 forever, and every channel would -// read as configured-but-not-started. +// The broker source is a function rather than a captured value because the +// connection is process state: the daemon's shared eventbus client in one +// composition, the Gateway's own task-actions connection in the other. type statusHealth struct { - broker func() (connected, ok bool) - containers func() (active, capacity int, ok bool) - channels func() (channels []gateway.ChannelHealth, ok bool) - chatModel *providerOutcomeRecorder - lastPoll func() (at time.Time, ok bool) + broker func() (connected, ok bool) + chatModel *providerOutcomeRecorder } func (s statusHealth) Health() gateway.HealthReport { @@ -104,58 +95,26 @@ func (s statusHealth) Health() gateway.HealthReport { report.Broker = &gateway.BrokerHealth{Connected: connected} } } - if s.containers != nil { - if active, capacity, ok := s.containers(); ok { - report.Containers = &gateway.ContainerHealth{Active: active, Cap: capacity} - } - } - if s.channels != nil { - if channels, ok := s.channels(); ok { - report.Channels = channels - } - } if s.chatModel != nil { outcome, attempted := s.chatModel.LastChatModelOutcome() report.ChatModel = &gateway.ChatModelHealth{Attempted: attempted, Outcome: outcome} } - if s.lastPoll != nil { - if at, ok := s.lastPoll(); ok { - report.LastPoll = &at - } - } return report } -// newStatusHealth builds the health source for this process. Sources whose -// subsystem is absent here (a deployment with no container pool, the Gateway -// process with no daemon) stay nil, and /status omits them. +// newStatusHealth builds the health source this process serves. It carries the +// two facts a Router in this process can render: the process's own broker +// connection, and the last chat-model call it made. A source whose subsystem +// is absent here stays nil, and /status omits that line. // -// The closures read boot's fields at call time on purpose. Gateways are wired -// before the daemon is built, so boot.d is nil when this runs -- capturing it -// here would leave the poll section permanently absent in the one process that -// actually polls, with nothing at runtime to show the wire was dead. +// The pool, the poll loop and the channel managers stay out even in the +// composition that holds them. setupGatewayChat is the only production +// constructor of a Router; it runs in the Gateway, which holds none of the +// three, so a producer reading them would build a section no /status reply can +// carry. func newStatusHealth(b *boot) gateway.HealthSource { - s := statusHealth{ + return statusHealth{ chatModel: b.providerOutcomes, - containers: func() (int, int, bool) { - if b.containerPool == nil { - return 0, 0, false - } - return b.containerPool.Active(), b.containerPool.Cap(), true - }, - // No channel source: the chat front-ends run in the extracted - // Messaging Service, so this process cannot observe a lifecycle it - // does not drive. Reporting anything here would be a guess, and - // "truthful or absent" says omit the section instead. Restoring it - // needs the Messaging Service to publish its channel lifecycle - // (archie-core-8cda.6.8). - channels: nil, - lastPoll: func() (time.Time, bool) { - if b.d == nil { - return time.Time{}, false - } - return b.d.LastPollAt(), true - }, broker: func() (bool, bool) { switch { case b.natsClient != nil: @@ -167,36 +126,4 @@ func newStatusHealth(b *boot) gateway.HealthSource { } }, } - return s -} - -// channelHealth projects the channel manager's snapshot into /status' view, so -// the gateway package does not depend on the channels package. It mirrors -// readiness.go's channelStates projection for the same reason. -// -// An unset display name falls back to the channel id: a line reading -// " running" would leave an operator unable to tell which channel is broken, -// which is the one thing this line exists to say. -// -// Detail is carried through whenever it is set, exactly as the dashboard reads -// it. It is one field serving two purposes -- a descriptor's standing -// configuration caveat ("token set, but the allowlist is empty") and a runtime -// failure reason -- and the manager keeps the last non-empty one, so a channel -// that failed and later recovered still shows the old reason. Filtering by -// state here would drop the caveat that matters most on a *running* channel -// (the empty allowlist one), so the split belongs at the producer, not here. -func channelHealth(m *status.Manager) []gateway.ChannelHealth { - if m == nil { - return nil - } - snapshot := m.Snapshot() - out := make([]gateway.ChannelHealth, 0, len(snapshot)) - for _, s := range snapshot { - out = append(out, gateway.ChannelHealth{ - Name: cmp.Or(s.Name, s.ID), - State: string(s.State), - Detail: s.Detail, - }) - } - return out } diff --git a/internal/app/archied/status_health_test.go b/internal/app/archied/status_health_test.go index b78dce541..fcc3ad68c 100644 --- a/internal/app/archied/status_health_test.go +++ b/internal/app/archied/status_health_test.go @@ -13,11 +13,12 @@ import ( "testing" "time" + natsio "github.com/nats-io/nats.go" "github.com/samcharles93/ai-sdk/chat" "github.com/samcharles93/ai-sdk/core" "github.com/samcharles93/archie-core/internal/agentexec" - "github.com/samcharles93/archie-core/internal/channels/status" + "github.com/samcharles93/archie-core/internal/container" "github.com/samcharles93/archie-core/internal/daemon" "github.com/samcharles93/archie-core/internal/domain/curator" "github.com/samcharles93/archie-core/internal/domain/messaging" @@ -210,21 +211,59 @@ func TestStatusHealthOmitsSourcesThisProcessDoesNotOwn(t *testing.T) { } } +// TestStatusHealthReportsOnlyBrokerAndChatModel pins what every /status +// surface actually serves. The only production router.Health is the Gateway's, +// and the facts the Gateway holds are the ones this boot is given here. A +// producer wired to the pool or the poll loop would build a section nothing in +// this process can serve -- the operator would read a full health list in a +// process that has no container pool, and would read it as measured. +func TestStatusHealthReportsOnlyBrokerAndChatModel(t *testing.T) { + srv := startEmbeddedNATS(t) + conn, err := natsio.Connect(srv.ClientURL()) + if err != nil { + t.Fatalf("nats connect: %v", err) + } + t.Cleanup(conn.Close) + + recorder := newProviderOutcomeRecorder() + recorder.record("openai/gpt-5.6", nil) + + // The pool and the daemon are present here on purpose: the daemon + // composition holds both, so a producer reading them would claim a + // container and poll line from a process whose router cannot serve it. + b := &boot{ + taskActionsConn: conn, + providerOutcomes: recorder, + containerPool: &container.Pool{}, + d: &daemon.Daemon{}, + } + + report := newStatusHealth(b).Health() + + if report.Broker == nil || !report.Broker.Connected { + t.Errorf("Broker = %+v, want this process's own connected broker connection", report.Broker) + } + if report.ChatModel == nil || !report.ChatModel.Attempted { + t.Errorf("ChatModel = %+v, want the recorded call", report.ChatModel) + } + if report.Containers != nil { + t.Errorf("Containers = %+v, want nil: no /status surface is served from this boot's pool", report.Containers) + } + if report.Channels != nil { + t.Errorf("Channels = %+v, want nil: no /status surface is served from this boot's channels", report.Channels) + } + if report.LastPoll != nil { + t.Errorf("LastPoll = %v, want nil: no /status surface is served from this boot's poll loop", report.LastPoll) + } +} + // TestStatusHealthReportsEveryWiredSource pins that each wired source reaches // the report, and that a source reporting "nothing has happened yet" stays // distinguishable from an absent source. func TestStatusHealthReportsEveryWiredSource(t *testing.T) { - polled := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC) recorder := newProviderOutcomeRecorder() recorder.record("openai/gpt-5.6", errors.New("401 Unauthorized")) - channels := status.NewManager([]status.Descriptor{ - {ID: "telegram", Name: "Telegram", Configured: true}, - {ID: "email", Configured: true}, - }) - channels.MarkRunning("telegram") - channels.MarkFailed("email", "dial tcp: connection refused") - tests := []struct { name string check func(*testing.T, gateway.HealthReport) @@ -239,33 +278,6 @@ func TestStatusHealthReportsEveryWiredSource(t *testing.T) { } }, }, - { - name: "container pool", - src: statusHealth{containers: func() (int, int, bool) { return 2, 4, true }}, - check: func(t *testing.T, r gateway.HealthReport) { - if r.Containers == nil || r.Containers.Active != 2 || r.Containers.Cap != 4 { - t.Errorf("Containers = %+v, want 2/4", r.Containers) - } - }, - }, - { - name: "channel manager", - src: statusHealth{channels: func() ([]gateway.ChannelHealth, bool) { return channelHealth(channels), true }}, - check: func(t *testing.T, r gateway.HealthReport) { - want := []gateway.ChannelHealth{ - {Name: "Telegram", State: "running"}, - {Name: "email", State: "failed", Detail: "dial tcp: connection refused"}, - } - if len(r.Channels) != len(want) { - t.Fatalf("Channels = %+v, want %+v", r.Channels, want) - } - for i := range want { - if r.Channels[i] != want[i] { - t.Errorf("Channels[%d] = %+v, want %+v", i, r.Channels[i], want[i]) - } - } - }, - }, { name: "chat model recorder", src: statusHealth{chatModel: recorder}, @@ -290,36 +302,6 @@ func TestStatusHealthReportsEveryWiredSource(t *testing.T) { } }, }, - { - name: "poll loop that has run", - src: statusHealth{lastPoll: func() (time.Time, bool) { return polled, true }}, - check: func(t *testing.T, r gateway.HealthReport) { - if r.LastPoll == nil || !r.LastPoll.Equal(polled) { - t.Errorf("LastPoll = %v, want %v", r.LastPoll, polled) - } - }, - }, - { - name: "poll loop that has not run a pass yet", - src: statusHealth{lastPoll: func() (time.Time, bool) { return time.Time{}, true }}, - check: func(t *testing.T, r gateway.HealthReport) { - if r.LastPoll == nil { - t.Fatal("LastPoll = nil, want a present \"not yet\" value") - } - if !r.LastPoll.IsZero() { - t.Errorf("LastPoll = %v, want the zero time", r.LastPoll) - } - }, - }, - { - name: "poll loop that is not this process's to report", - src: statusHealth{lastPoll: func() (time.Time, bool) { return polled, false }}, - check: func(t *testing.T, r gateway.HealthReport) { - if r.LastPoll != nil { - t.Errorf("LastPoll = %v, want nil", r.LastPoll) - } - }, - }, } for _, tc := range tests { @@ -329,81 +311,6 @@ func TestStatusHealthReportsEveryWiredSource(t *testing.T) { } } -// TestStatusHealthReadsDeferredSourcesAtCallTime pins the composition-order -// trap this source sits behind: the daemon is built AFTER the gateways -// (main.go runs setupGateways before buildDaemon), so the health source is -// constructed while boot.d is still nil. A source that captured boot.d at -// construction would report the poll section as permanently absent -- /status -// would look wired and never say anything about polling. -func TestStatusHealthReadsDeferredSourcesAtCallTime(t *testing.T) { - b := &boot{} - src := newStatusHealth(b) - - if got := src.Health().LastPoll; got != nil { - t.Fatalf("LastPoll = %v with no daemon yet, want nil", got) - } - - b.d = &daemon.Daemon{} - - got := src.Health().LastPoll - if got == nil { - t.Fatal("LastPoll = nil after the daemon was built, want the source to read boot.d at call time") - } - if !got.IsZero() { - t.Errorf("LastPoll = %v, want the zero time: no poll pass has run yet", got) - } -} - -// TestChannelHealthProjectsManagerSnapshot pins the mapping from the channel -// manager's own state to what /status renders -- including the two edges an -// operator would otherwise have to guess at: a channel with no display name -// shows its id, and an unconfigured channel is reported as such rather than -// omitted. -func TestChannelHealthProjectsManagerSnapshot(t *testing.T) { - manager := status.NewManager([]status.Descriptor{ - {ID: "telegram", Name: "Telegram", Configured: true}, - {ID: "email", Configured: true}, - {ID: "webhook"}, - }) - manager.MarkStarting("telegram") - manager.MarkRunning("telegram") - manager.MarkFailed("email", "listen tcp: address already in use") - - tests := []struct { - name string - manager *status.Manager - want []gateway.ChannelHealth - }{ - { - name: "no channel manager", - manager: nil, - want: nil, - }, - { - name: "manager snapshot", - manager: manager, - want: []gateway.ChannelHealth{ - {Name: "Telegram", State: "running"}, - {Name: "email", State: "failed", Detail: "listen tcp: address already in use"}, - {Name: "webhook", State: "stopped"}, - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := channelHealth(tc.manager) - if len(got) != len(tc.want) { - t.Fatalf("channelHealth() = %+v, want %+v", got, tc.want) - } - for i := range tc.want { - if got[i] != tc.want[i] { - t.Errorf("channelHealth()[%d] = %+v, want %+v", i, got[i], tc.want[i]) - } - } - }) - } -} - // TestSetupGatewayChatCarriesStatusHealth pins the last hop: the router the // composition root builds must expose the health source, or /status answers // from the queue and runtime sections alone while the composition root