fix(webui): retry event pump priming until the state store answers
archie-ui binds before archie-state-store is guaranteed to be listening, so the first prime is routinely refused. PumpEvents returned on that error, which left the activity feed history-only for the life of the process. Prime now retries with capped backoff and gives up only when the context ends. Orders archie-ui.service after archie-state-store.service as well, which narrows the window without closing it: After= sequences the start, it does not wait for the listener.
This commit is contained in:
1 parent
d3f72a4b20
commit
13d32f34b0
3 files changed
+101
-2
No files matched your search
@@ -97,7 +97,8 @@ the dashboard's health endpoint behind the token. A watchdog can curl
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Archie UI Service
|
||||
After=network.target
|
||||
After=network.target archie-state-store.service
|
||||
Wants=archie-state-store.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
@@ -121,6 +122,11 @@ GATEWAY_TOKEN=... # presented to archie-gateway
|
||||
STATE_STORE_TOKEN=... # presented to archie-state-store
|
||||
```
|
||||
|
||||
`After=` orders the start, it does not wait for the store to bind. The UI
|
||||
process closes that gap itself: its event pump retries priming with backoff
|
||||
until the State Store answers, so the activity feed goes live once the store
|
||||
is up rather than staying history-only until the next restart.
|
||||
|
||||
```ini
|
||||
ExecStart=%h/.local/bin/archie-ui -config %h/.config/archie/config.toml -token-file %h/.local/share/archie/web-token
|
||||
```
|
||||
|
||||
@@ -22,6 +22,14 @@ const (
|
||||
eventPumpPageSize = 200
|
||||
)
|
||||
|
||||
// Backoff bounds for priming. The UI process and the State Store are separate
|
||||
// units with no ordering between them, so a refused dial at startup is
|
||||
// ordinary rather than fatal and the pump waits the store out.
|
||||
const (
|
||||
eventPumpPrimeRetryMin = 250 * time.Millisecond
|
||||
eventPumpPrimeRetryMax = 30 * time.Second
|
||||
)
|
||||
|
||||
// eventPump is the UI process's substitute for the daemon's in-process event
|
||||
// bus. The daemon hands persisted events to Server.Broadcast from its own bus
|
||||
// subscription (internal/app/archied/main.go's persistAndBroadcastEvents); a
|
||||
@@ -42,6 +50,36 @@ type eventPump struct {
|
||||
// watermark is the highest event id already handed to broadcast. It only
|
||||
// ever moves forward, and only past an event that was delivered.
|
||||
watermark int64
|
||||
// primeRetryMin is the first backoff between priming attempts. Zero
|
||||
// means eventPumpPrimeRetryMin; a test sets it small.
|
||||
primeRetryMin time.Duration
|
||||
}
|
||||
|
||||
// primeWithRetry primes, retrying with capped exponential backoff until it
|
||||
// succeeds or ctx ends. Returning on the first error instead left the
|
||||
// activity feed history-only for the life of the process whenever the UI won
|
||||
// the startup race against the State Store.
|
||||
func (p *eventPump) primeWithRetry(ctx context.Context) error {
|
||||
delay := p.primeRetryMin
|
||||
if delay <= 0 {
|
||||
delay = eventPumpPrimeRetryMin
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
err := p.prime(ctx)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return err
|
||||
}
|
||||
p.log("event pump priming failed; retrying", "attempt", attempt, "retry_in", delay, "err", err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return err
|
||||
case <-time.After(delay):
|
||||
}
|
||||
delay = min(delay*2, eventPumpPrimeRetryMax)
|
||||
}
|
||||
}
|
||||
|
||||
// eventReader is the one method the pump needs, so a test can drive it
|
||||
@@ -65,7 +103,7 @@ func (s *Server) PumpEvents(ctx context.Context, interval time.Duration) error {
|
||||
interval = DefaultEventPollInterval
|
||||
}
|
||||
pump := s.newEventPump()
|
||||
if err := pump.prime(ctx); err != nil {
|
||||
if err := pump.primeWithRetry(ctx); err != nil {
|
||||
return fmt.Errorf("prime event pump: %w", err)
|
||||
}
|
||||
pump.run(ctx, interval)
|
||||
|
||||
@@ -3,6 +3,7 @@ package webui
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -162,3 +163,57 @@ func openSSEStream(t *testing.T, ctx context.Context, ts *httptest.Server) <-cha
|
||||
}()
|
||||
return lines
|
||||
}
|
||||
|
||||
// flakyEventReader fails its first failures calls, then answers an empty
|
||||
// table. It stands in for the State Store during the window where the UI
|
||||
// process is up and the store has not bound its listener yet.
|
||||
type flakyEventReader struct {
|
||||
failures int
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *flakyEventReader) EventsSince(context.Context, int64, int) ([]events.Event, error) {
|
||||
f.calls++
|
||||
if f.calls <= f.failures {
|
||||
return nil, errors.New("connection refused")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// archie-ui binds before archie-state-store is guaranteed to be listening, so
|
||||
// the first prime is routinely refused. Returning on that error left the
|
||||
// activity feed history-only until the process was restarted by hand.
|
||||
func TestPumpPrimingRetriesUntilTheStoreAnswers(t *testing.T) {
|
||||
reader := &flakyEventReader{failures: 3}
|
||||
pump := &eventPump{
|
||||
store: reader,
|
||||
log: func(string, ...any) {},
|
||||
broadcast: func(events.Event) {},
|
||||
primeRetryMin: time.Millisecond,
|
||||
}
|
||||
|
||||
if err := pump.primeWithRetry(t.Context()); err != nil {
|
||||
t.Fatalf("primeWithRetry: %v", err)
|
||||
}
|
||||
if reader.calls != 4 {
|
||||
t.Fatalf("EventsSince called %d times, want 4 (3 refusals then success)", reader.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// A shutdown during the retry wait must end the pump, not hold the process
|
||||
// open until the store appears.
|
||||
func TestPumpPrimingStopsRetryingWhenTheContextEnds(t *testing.T) {
|
||||
reader := &flakyEventReader{failures: 1 << 30}
|
||||
pump := &eventPump{
|
||||
store: reader,
|
||||
log: func(string, ...any) {},
|
||||
broadcast: func(events.Event) {},
|
||||
primeRetryMin: time.Millisecond,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := pump.primeWithRetry(ctx); err == nil {
|
||||
t.Fatal("primeWithRetry returned no error after the context ended")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user