diff --git a/deployments/systemd-user-service.md b/deployments/systemd-user-service.md index e8e94515b..9e78729ba 100644 --- a/deployments/systemd-user-service.md +++ b/deployments/systemd-user-service.md @@ -97,7 +97,8 @@ the dashboard's health endpoint behind the token. A watchdog can curl ```ini [Unit] Description=Archie UI Service -After=network.target +After=network.target archie-state-store.service +Wants=archie-state-store.service [Service] Type=simple @@ -121,6 +122,11 @@ GATEWAY_TOKEN=... # presented to archie-gateway STATE_STORE_TOKEN=... # presented to archie-state-store ``` +`After=` orders the start, it does not wait for the store to bind. The UI +process closes that gap itself: its event pump retries priming with backoff +until the State Store answers, so the activity feed goes live once the store +is up rather than staying history-only until the next restart. + ```ini ExecStart=%h/.local/bin/archie-ui -config %h/.config/archie/config.toml -token-file %h/.local/share/archie/web-token ``` diff --git a/internal/webui/event_pump.go b/internal/webui/event_pump.go index 3f7d46658..ce3d35600 100644 --- a/internal/webui/event_pump.go +++ b/internal/webui/event_pump.go @@ -22,6 +22,14 @@ const ( eventPumpPageSize = 200 ) +// Backoff bounds for priming. The UI process and the State Store are separate +// units with no ordering between them, so a refused dial at startup is +// ordinary rather than fatal and the pump waits the store out. +const ( + eventPumpPrimeRetryMin = 250 * time.Millisecond + eventPumpPrimeRetryMax = 30 * time.Second +) + // eventPump is the UI process's substitute for the daemon's in-process event // bus. The daemon hands persisted events to Server.Broadcast from its own bus // subscription (internal/app/archied/main.go's persistAndBroadcastEvents); a @@ -42,6 +50,36 @@ type eventPump struct { // watermark is the highest event id already handed to broadcast. It only // ever moves forward, and only past an event that was delivered. watermark int64 + // primeRetryMin is the first backoff between priming attempts. Zero + // means eventPumpPrimeRetryMin; a test sets it small. + primeRetryMin time.Duration +} + +// primeWithRetry primes, retrying with capped exponential backoff until it +// succeeds or ctx ends. Returning on the first error instead left the +// activity feed history-only for the life of the process whenever the UI won +// the startup race against the State Store. +func (p *eventPump) primeWithRetry(ctx context.Context) error { + delay := p.primeRetryMin + if delay <= 0 { + delay = eventPumpPrimeRetryMin + } + for attempt := 1; ; attempt++ { + err := p.prime(ctx) + if err == nil { + return nil + } + if ctx.Err() != nil { + return err + } + p.log("event pump priming failed; retrying", "attempt", attempt, "retry_in", delay, "err", err) + select { + case <-ctx.Done(): + return err + case <-time.After(delay): + } + delay = min(delay*2, eventPumpPrimeRetryMax) + } } // eventReader is the one method the pump needs, so a test can drive it @@ -65,7 +103,7 @@ func (s *Server) PumpEvents(ctx context.Context, interval time.Duration) error { interval = DefaultEventPollInterval } pump := s.newEventPump() - if err := pump.prime(ctx); err != nil { + if err := pump.primeWithRetry(ctx); err != nil { return fmt.Errorf("prime event pump: %w", err) } pump.run(ctx, interval) diff --git a/internal/webui/event_pump_test.go b/internal/webui/event_pump_test.go index 181d01a1d..fcf9812ac 100644 --- a/internal/webui/event_pump_test.go +++ b/internal/webui/event_pump_test.go @@ -3,6 +3,7 @@ package webui import ( "bufio" "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -162,3 +163,57 @@ func openSSEStream(t *testing.T, ctx context.Context, ts *httptest.Server) <-cha }() return lines } + +// flakyEventReader fails its first failures calls, then answers an empty +// table. It stands in for the State Store during the window where the UI +// process is up and the store has not bound its listener yet. +type flakyEventReader struct { + failures int + calls int +} + +func (f *flakyEventReader) EventsSince(context.Context, int64, int) ([]events.Event, error) { + f.calls++ + if f.calls <= f.failures { + return nil, errors.New("connection refused") + } + return nil, nil +} + +// archie-ui binds before archie-state-store is guaranteed to be listening, so +// the first prime is routinely refused. Returning on that error left the +// activity feed history-only until the process was restarted by hand. +func TestPumpPrimingRetriesUntilTheStoreAnswers(t *testing.T) { + reader := &flakyEventReader{failures: 3} + pump := &eventPump{ + store: reader, + log: func(string, ...any) {}, + broadcast: func(events.Event) {}, + primeRetryMin: time.Millisecond, + } + + if err := pump.primeWithRetry(t.Context()); err != nil { + t.Fatalf("primeWithRetry: %v", err) + } + if reader.calls != 4 { + t.Fatalf("EventsSince called %d times, want 4 (3 refusals then success)", reader.calls) + } +} + +// A shutdown during the retry wait must end the pump, not hold the process +// open until the store appears. +func TestPumpPrimingStopsRetryingWhenTheContextEnds(t *testing.T) { + reader := &flakyEventReader{failures: 1 << 30} + pump := &eventPump{ + store: reader, + log: func(string, ...any) {}, + broadcast: func(events.Event) {}, + primeRetryMin: time.Millisecond, + } + + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + if err := pump.primeWithRetry(ctx); err == nil { + t.Fatal("primeWithRetry returned no error after the context ended") + } +}