EventsSince no longer depends on events.id being monotonic. The feed now pages a total order over (at, id): at is a fixed-width, lexicographically sortable UTC timestamp (storecontract.EventCursorLayout) and id is only a deterministic tie-break for rows sharing one timestamp. The cursor is an opaque string on the wire (EventsSinceRequest.cursor), carried verbatim as the SSE id: field and echoed back via Last-Event-ID, so EventSource resume needs no client change. A new idempotent migration normalises pre-existing RFC3339Nano at values to the fixed-width layout before finishTaskMigration, and idx_events_at backs the new query. Legacy integer and garbage cursors degrade to "from the beginning", preserving the old failed-ParseInt-yields-0 behaviour. Amends the "Dashboard live event delivery" decision's cursor-cannot-skip justification, which previously rested on commit-order id assignment.
183 lines
6.2 KiB
Go
183 lines
6.2 KiB
Go
package webui
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/samcharles93/archie-core/internal/domain/storecontract"
|
|
"github.com/samcharles93/archie-core/internal/events"
|
|
)
|
|
|
|
// DefaultEventPollInterval is how often the UI process asks the State Store
|
|
// for events it has not seen. It bounds activity-feed latency; the feed
|
|
// reports task lifecycle history, and an operator action already gets its own
|
|
// synchronous HTTP response, so a second is not felt.
|
|
const DefaultEventPollInterval = time.Second
|
|
|
|
// Page sizes. Priming walks the whole table once and wants few round trips;
|
|
// steady-state ticks expect nothing or a handful and want small messages.
|
|
const (
|
|
eventPumpPrimePageSize = 5000
|
|
eventPumpPageSize = 200
|
|
)
|
|
|
|
// Backoff bounds for priming. The UI process and the State Store are separate
|
|
// units with no ordering between them, so a refused dial at startup is
|
|
// ordinary rather than fatal and the pump waits the store out.
|
|
const (
|
|
eventPumpPrimeRetryMin = 250 * time.Millisecond
|
|
eventPumpPrimeRetryMax = 30 * time.Second
|
|
)
|
|
|
|
// eventPump is the UI process's substitute for the daemon's in-process event
|
|
// bus. The daemon hands persisted events to Server.Broadcast from its own bus
|
|
// subscription (internal/app/archied/main.go's persistAndBroadcastEvents); a
|
|
// process that owns no bus reads the same events back out of the State Store
|
|
// instead, over the EventsSince cursor.
|
|
//
|
|
// Polling is sufficient rather than a compromise, for the reasons recorded in
|
|
// docs/architecture/migration-decisions.md ("Dashboard live event delivery"):
|
|
// EventsSince pages a total order over (at, id) with a fixed-width time key
|
|
// and a deterministic id tie-break, so the cursor cannot skip or repeat;
|
|
// sseStream.drain treats a broadcast as a wakeup and re-reads the gap from the
|
|
// store, so nothing here changes SSE semantics; and reading the table catches
|
|
// writers that never touch a bus, including the audit event ArchiveTask writes
|
|
// inside its transaction.
|
|
type eventPump struct {
|
|
store eventReader
|
|
log func(string, ...any)
|
|
broadcast func(events.Event)
|
|
// watermark is the highest event cursor already handed to broadcast. It only
|
|
// ever moves forward, and only past an event that was delivered.
|
|
watermark string
|
|
// primeRetryMin is the first backoff between priming attempts. Zero
|
|
// means eventPumpPrimeRetryMin; a test sets it small.
|
|
primeRetryMin time.Duration
|
|
}
|
|
|
|
// primeWithRetry primes, retrying with capped exponential backoff until it
|
|
// succeeds or ctx ends. Returning on the first error instead left the
|
|
// activity feed history-only for the life of the process whenever the UI won
|
|
// the startup race against the State Store.
|
|
func (p *eventPump) primeWithRetry(ctx context.Context) error {
|
|
delay := p.primeRetryMin
|
|
if delay <= 0 {
|
|
delay = eventPumpPrimeRetryMin
|
|
}
|
|
for attempt := 1; ; attempt++ {
|
|
err := p.prime(ctx)
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if ctx.Err() != nil {
|
|
return err
|
|
}
|
|
p.log("event pump priming failed; retrying", "attempt", attempt, "retry_in", delay, "err", err)
|
|
select {
|
|
case <-ctx.Done():
|
|
return err
|
|
case <-time.After(delay):
|
|
}
|
|
delay = min(delay*2, eventPumpPrimeRetryMax)
|
|
}
|
|
}
|
|
|
|
// eventReader is the one method the pump needs, so a test can drive it
|
|
// without a whole TaskStore.
|
|
type eventReader interface {
|
|
EventsSince(ctx context.Context, cursor string, limit int) ([]events.Event, error)
|
|
}
|
|
|
|
func (s *Server) newEventPump() *eventPump {
|
|
return &eventPump{store: s.Store, log: s.logf, broadcast: s.Broadcast}
|
|
}
|
|
|
|
// PumpEvents delivers events persisted after it starts to every connected SSE
|
|
// client, until ctx ends. It is the UI process's only live-delivery path;
|
|
// history is served by each client's own catch-up, not by this.
|
|
func (s *Server) PumpEvents(ctx context.Context, interval time.Duration) error {
|
|
if s.Store == nil {
|
|
return errors.New("event pump needs a store to read events from")
|
|
}
|
|
if interval <= 0 {
|
|
interval = DefaultEventPollInterval
|
|
}
|
|
pump := s.newEventPump()
|
|
if err := pump.primeWithRetry(ctx); err != nil {
|
|
return fmt.Errorf("prime event pump: %w", err)
|
|
}
|
|
pump.run(ctx, interval)
|
|
return nil
|
|
}
|
|
|
|
// prime advances the watermark past everything already persisted, without
|
|
// delivering any of it. Without this the pump would replay the entire events
|
|
// table into every browser the first time it ticks; catch-up already serves a
|
|
// client the history it asked for.
|
|
func (p *eventPump) prime(ctx context.Context) error {
|
|
for {
|
|
batch, err := p.store.EventsSince(ctx, p.watermark, eventPumpPrimePageSize)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
before := p.watermark
|
|
for _, e := range batch {
|
|
if cursor := storecontract.EventCursor(e.At, e.ID); cursor > p.watermark {
|
|
p.watermark = cursor
|
|
}
|
|
}
|
|
// A short page is the end of the table. The watermark check is the
|
|
// guard against a full page that advanced nothing, which would
|
|
// otherwise spin forever.
|
|
if len(batch) < eventPumpPrimePageSize || p.watermark == before {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// deliver broadcasts every event after the watermark and reports how many.
|
|
// A fetch that fails leaves the watermark where it was, so the next tick
|
|
// retries the same range: a State Store restart costs latency, not events.
|
|
func (p *eventPump) deliver(ctx context.Context) (int, error) {
|
|
delivered := 0
|
|
for {
|
|
batch, err := p.store.EventsSince(ctx, p.watermark, eventPumpPageSize)
|
|
if err != nil {
|
|
return delivered, err
|
|
}
|
|
before := p.watermark
|
|
for _, e := range batch {
|
|
cursor := storecontract.EventCursor(e.At, e.ID)
|
|
if cursor <= p.watermark {
|
|
continue
|
|
}
|
|
p.broadcast(e)
|
|
p.watermark = cursor
|
|
delivered++
|
|
}
|
|
if len(batch) < eventPumpPageSize || p.watermark == before {
|
|
return delivered, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// run ticks until ctx ends. A failed tick is logged and retried on the next
|
|
// one rather than ending the pump: the feed is a view, and losing it for a
|
|
// poll interval is better than losing it for the life of the process.
|
|
func (p *eventPump) run(ctx context.Context, interval time.Duration) {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if _, err := p.deliver(ctx); err != nil && ctx.Err() == nil {
|
|
p.log("event pump fetch failed", "since", p.watermark, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|