fix(webui): retry event pump priming until the state store answers

archie-ui binds before archie-state-store is guaranteed to be listening, so
the first prime is routinely refused. PumpEvents returned on that error, which
left the activity feed history-only for the life of the process. Prime now
retries with capped backoff and gives up only when the context ends.

Orders archie-ui.service after archie-state-store.service as well, which
narrows the window without closing it: After= sequences the start, it does
not wait for the listener.
This commit is contained in:
sam committed 2026-09-18 23:59:40 +10:00
1 parent d3f72a4b20
commit 13d32f34b0
3 files changed
+101 -2

No files matched your search

+7 -1
View File
@@ -97,7 +97,8 @@ the dashboard's health endpoint behind the token. A watchdog can curl
```ini
[Unit]
Description=Archie UI Service
After=network.target
After=network.target archie-state-store.service
Wants=archie-state-store.service
[Service]
Type=simple
@@ -121,6 +122,11 @@ GATEWAY_TOKEN=... # presented to archie-gateway
STATE_STORE_TOKEN=... # presented to archie-state-store
```
`After=` orders the start, it does not wait for the store to bind. The UI
process closes that gap itself: its event pump retries priming with backoff
until the State Store answers, so the activity feed goes live once the store
is up rather than staying history-only until the next restart.
```ini
ExecStart=%h/.local/bin/archie-ui -config %h/.config/archie/config.toml -token-file %h/.local/share/archie/web-token
```
+39 -1
View File
@@ -22,6 +22,14 @@ const (
eventPumpPageSize = 200
)
// Backoff bounds for priming. The UI process and the State Store are separate
// units with no ordering between them, so a refused dial at startup is
// ordinary rather than fatal and the pump waits the store out.
const (
eventPumpPrimeRetryMin = 250 * time.Millisecond
eventPumpPrimeRetryMax = 30 * time.Second
)
// eventPump is the UI process's substitute for the daemon's in-process event
// bus. The daemon hands persisted events to Server.Broadcast from its own bus
// subscription (internal/app/archied/main.go's persistAndBroadcastEvents); a
@@ -42,6 +50,36 @@ type eventPump struct {
// watermark is the highest event id already handed to broadcast. It only
// ever moves forward, and only past an event that was delivered.
watermark int64
// primeRetryMin is the first backoff between priming attempts. Zero
// means eventPumpPrimeRetryMin; a test sets it small.
primeRetryMin time.Duration
}
// primeWithRetry primes, retrying with capped exponential backoff until it
// succeeds or ctx ends. Returning on the first error instead left the
// activity feed history-only for the life of the process whenever the UI won
// the startup race against the State Store.
func (p *eventPump) primeWithRetry(ctx context.Context) error {
delay := p.primeRetryMin
if delay <= 0 {
delay = eventPumpPrimeRetryMin
}
for attempt := 1; ; attempt++ {
err := p.prime(ctx)
if err == nil {
return nil
}
if ctx.Err() != nil {
return err
}
p.log("event pump priming failed; retrying", "attempt", attempt, "retry_in", delay, "err", err)
select {
case <-ctx.Done():
return err
case <-time.After(delay):
}
delay = min(delay*2, eventPumpPrimeRetryMax)
}
}
// eventReader is the one method the pump needs, so a test can drive it
@@ -65,7 +103,7 @@ func (s *Server) PumpEvents(ctx context.Context, interval time.Duration) error {
interval = DefaultEventPollInterval
}
pump := s.newEventPump()
if err := pump.prime(ctx); err != nil {
if err := pump.primeWithRetry(ctx); err != nil {
return fmt.Errorf("prime event pump: %w", err)
}
pump.run(ctx, interval)
+55
View File
@@ -3,6 +3,7 @@ package webui
import (
"bufio"
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
@@ -162,3 +163,57 @@ func openSSEStream(t *testing.T, ctx context.Context, ts *httptest.Server) <-cha
}()
return lines
}
// flakyEventReader fails its first failures calls, then answers an empty
// table. It stands in for the State Store during the window where the UI
// process is up and the store has not bound its listener yet.
type flakyEventReader struct {
failures int
calls int
}
func (f *flakyEventReader) EventsSince(context.Context, int64, int) ([]events.Event, error) {
f.calls++
if f.calls <= f.failures {
return nil, errors.New("connection refused")
}
return nil, nil
}
// archie-ui binds before archie-state-store is guaranteed to be listening, so
// the first prime is routinely refused. Returning on that error left the
// activity feed history-only until the process was restarted by hand.
func TestPumpPrimingRetriesUntilTheStoreAnswers(t *testing.T) {
reader := &flakyEventReader{failures: 3}
pump := &eventPump{
store: reader,
log: func(string, ...any) {},
broadcast: func(events.Event) {},
primeRetryMin: time.Millisecond,
}
if err := pump.primeWithRetry(t.Context()); err != nil {
t.Fatalf("primeWithRetry: %v", err)
}
if reader.calls != 4 {
t.Fatalf("EventsSince called %d times, want 4 (3 refusals then success)", reader.calls)
}
}
// A shutdown during the retry wait must end the pump, not hold the process
// open until the store appears.
func TestPumpPrimingStopsRetryingWhenTheContextEnds(t *testing.T) {
reader := &flakyEventReader{failures: 1 << 30}
pump := &eventPump{
store: reader,
log: func(string, ...any) {},
broadcast: func(events.Event) {},
primeRetryMin: time.Millisecond,
}
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
defer cancel()
if err := pump.primeWithRetry(ctx); err == nil {
t.Fatal("primeWithRetry returned no error after the context ended")
}
}