Merge pull request #463 from samcharles93/fix/gateway-session-defects

fix(gateway): harden session recovery paths
This commit is contained in:
sam authored and GitHub committed 2026-08-09 23:24:05 +10:00
commit 283e85ba7a
10 files changed
+570 -27

No files matched your search

+20
View File
@@ -3,8 +3,28 @@ package gateway
import (
"testing"
"time"
"github.com/google/uuid"
)
func TestPriorReplyRecognisesLegacyCanonicalMessageID(t *testing.T) {
t.Parallel()
const (
sessionID = "a\x00b"
sourceID = "source"
identity = "archie"
)
legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
history := []Message{
{MessageID: legacyID, SourceID: sourceID, From: "alice", Text: "question"},
{From: identity, Text: "answer"},
}
if got := PriorReply(history, sessionID, sourceID, identity); got != "answer" {
t.Fatalf("PriorReply = %q, want legacy reply", got)
}
}
// Making the store idempotent only stopped the duplicate *record*. The chat
// path saved the inbound message, ignored the fact that nothing changed, and
// went on to call the model again -- a second billed turn, any tool side
+30
View File
@@ -462,6 +462,36 @@ func TestRouteUndoRemovesMultiple(t *testing.T) {
}
}
func TestRouteUndoReportsActualPartialRemoval(t *testing.T) {
r := newTestRouter("telegram")
ctx := t.Context()
const sessionID = "test-undo-partial"
sc := SessionContext{
SessionID: sessionID,
Source: SessionSource{Platform: "telegram", BotUser: "archie", ChannelID: "chat-partial"},
CreatedAt: time.Now(), LastActiveAt: time.Now(),
}
if err := r.sessionTracker.sessions.Save(ctx, sc); err != nil {
t.Fatalf("Save session: %v", err)
}
for i := range 6 {
if err := r.sessionTracker.sessions.SaveMessage(ctx, sessionID, Message{
From: "alice", Text: fmt.Sprintf("msg%d", i),
}); err != nil {
t.Fatalf("SaveMessage(%d): %v", i, err)
}
}
r.sessionTracker.setActive("chat-partial", "", sessionID)
reply, err := r.Route(ctx, Message{Text: "/undo 10", ChannelID: "chat-partial"})
if err != nil {
t.Fatalf("Route: %v", err)
}
if !strings.Contains(reply, "Removed 6 messages") {
t.Fatalf("reply = %q, want actual removal count", reply)
}
}
func TestRouteUndoNoMessages(t *testing.T) {
r := newTestRouter("telegram")
+33 -6
View File
@@ -2,6 +2,7 @@ package gateway
import (
"context"
"encoding/binary"
"strings"
"time"
@@ -101,16 +102,42 @@ func sessionRecency(sc SessionContext) time.Time {
// messageIDNamespace scopes deterministic canonical message identifiers.
var messageIDNamespace = uuid.MustParse("6f8d2b1e-9c4a-4f37-8a56-1d0e7b3c9a42")
// messageIDNamespaceV2 isolates the injective encoding used for inputs that
// contain the legacy separator. Ordinary platform/session IDs keep their
// existing canonical IDs, so an upgrade cannot duplicate redelivered messages.
var messageIDNamespaceV2 = uuid.MustParse("e6ac7868-cf49-4e65-a5a1-a1549fd12f67")
func newMessageID(sessionID, sourceID string) string {
if sourceID == "" {
return uuid.NewString()
}
return uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
current, _ := canonicalMessageIDs(sessionID, sourceID)
return current
}
func canonicalMessageIDs(sessionID, sourceID string) (current, legacy string) {
legacy = uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
if strings.ContainsRune(sessionID, '\x00') || strings.ContainsRune(sourceID, '\x00') {
return uuid.NewSHA1(messageIDNamespaceV2, encodeCanonicalPair(sessionID, sourceID)).String(), legacy
}
return legacy, ""
}
func encodeCanonicalPair(first, second string) []byte {
encoded := make([]byte, 8+len(first)+len(second))
binary.BigEndian.PutUint64(encoded, uint64(len(first)))
copy(encoded[8:], first)
copy(encoded[8+len(first):], second)
return encoded
}
func stamp(msg Message) time.Time {
if msg.At.IsZero() {
return time.Now().UTC()
now := time.Now().UTC()
if msg.At.IsZero() || msg.At.After(now) {
// At orders the canonical conversation; it is not an authoritative
// copy of a platform clock. Persisting a future platform timestamp
// would force every later message beyond it to preserve monotonicity.
return now
}
return msg.At.UTC()
}
@@ -125,12 +152,12 @@ func CanonicalMessageID(sessionID, sourceID string) string {
// PriorReply returns the reply already produced for an upstream message.
func PriorReply(history []Message, sessionID, sourceID, identity string) string {
id := CanonicalMessageID(sessionID, sourceID)
if id == "" {
if sourceID == "" {
return ""
}
id, legacyID := canonicalMessageIDs(sessionID, sourceID)
for i, message := range history {
if message.MessageID != id {
if message.MessageID != id && message.MessageID != legacyID {
continue
}
if i+1 < len(history) && history[i+1].From == identity &&
+67 -4
View File
@@ -151,9 +151,43 @@ func openSQLiteSessionStore(dsn string) (SessionStore, error) {
if _, err := db.ExecContext(ctx, sqliteSessionSchema); err != nil {
return nil, errors.Join(fmt.Errorf("sessionstore: init schema: %w", err), db.Close())
}
if err := repairFutureMessageTimestamps(ctx, db, time.Now().UTC()); err != nil {
return nil, errors.Join(err, db.Close())
}
return &sqliteSessionStore{db: db}, nil
}
// repairFutureMessageTimestamps heals sessions poisoned by older versions
// that accepted a platform timestamp arbitrarily far ahead of the daemon's
// clock. Future rows retain their order and are rebased immediately after the
// session's last non-future row. With no earlier row, they end at now. A dense
// history can finish a few milliseconds beyond now, but never preserves the
// original hours- or days-long skew.
func repairFutureMessageTimestamps(ctx context.Context, db *sql.DB, now time.Time) error {
cutoff := now.UnixMilli()
_, err := db.ExecContext(ctx, `
WITH future AS MATERIALIZED (
SELECT m.id,
COALESCE(
(SELECT MAX(p.ts) FROM messages p
WHERE p.session_id = m.session_id AND p.ts <= ?),
? - COUNT(*) OVER (PARTITION BY m.session_id)
) + ROW_NUMBER() OVER (
PARTITION BY m.session_id ORDER BY m.ts, m.id
) AS repaired_ts
FROM messages m
WHERE m.ts > ?
)
UPDATE messages
SET ts = (SELECT repaired_ts FROM future WHERE future.id = messages.id)
WHERE id IN (SELECT id FROM future) AND ts > ?`,
cutoff, cutoff, cutoff, cutoff)
if err != nil {
return fmt.Errorf("sessionstore: repair future message timestamps: %w", err)
}
return nil
}
func (s *sqliteSessionStore) Close() error {
return s.db.Close()
}
@@ -579,8 +613,34 @@ func saveMessage(ctx context.Context, ex execer, sessionID string, msg Message)
// placing records deliberately.
func saveMessageAt(ctx context.Context, ex execer, sessionID string, msg Message, clamp bool) (string, error) {
id := msg.MessageID
if id == "" {
id = newMessageID(sessionID, msg.SourceID)
legacyID := ""
if msg.SourceID == "" {
if id == "" {
id = newMessageID(sessionID, "")
}
} else {
currentID, compatibleLegacyID := canonicalMessageIDs(sessionID, msg.SourceID)
if id == "" {
id = currentID
}
// TurnRunner supplies the current canonical ID before calling the
// store. Still check the compatible legacy identity: a crash can
// leave the old message durable before its turn record names it.
if id == currentID {
legacyID = compatibleLegacyID
}
}
if legacyID != "" {
var existingID string
err := ex.QueryRowContext(ctx,
`SELECT message_id FROM messages WHERE session_id = ? AND message_id = ?`,
sessionID, legacyID).Scan(&existingID)
switch {
case err == nil:
return existingID, nil
case !errors.Is(err, sql.ErrNoRows):
return "", fmt.Errorf("sessionstore: read legacy message identity: %w", err)
}
}
at := stamp(msg)
if clamp {
@@ -710,10 +770,13 @@ func (s *sqliteSessionStore) FindPriorReply(ctx context.Context, sessionID, sour
s.mu.Lock()
defer s.mu.Unlock()
currentID, legacyID := canonicalMessageIDs(sessionID, sourceID)
var ts int64
err := s.db.QueryRowContext(ctx, `
SELECT ts FROM messages WHERE session_id = ? AND message_id = ?`,
sessionID, CanonicalMessageID(sessionID, sourceID)).Scan(&ts)
SELECT ts FROM messages
WHERE session_id = ? AND message_id IN (?, ?)
ORDER BY CASE WHEN message_id = ? THEN 0 ELSE 1 END
LIMIT 1`, sessionID, currentID, legacyID, legacyID).Scan(&ts)
if errors.Is(err, sql.ErrNoRows) {
return "", nil
}
@@ -9,6 +9,8 @@ import (
"sync"
"testing"
"time"
"github.com/google/uuid"
)
func newTestSQLiteStore(t *testing.T) SessionStore {
@@ -93,6 +95,217 @@ func TestMessageIDsAreUnique(t *testing.T) {
}
}
func TestCanonicalMessageIDComponentsAreInjectivelyEncoded(t *testing.T) {
t.Parallel()
tests := []struct {
name string
firstSession string
firstSource string
secondSession string
secondSource string
}{
{
name: "separator can occur in session",
firstSession: "a\x00b",
firstSource: "c",
secondSession: "a",
secondSource: "b\x00c",
},
{
name: "empty session does not alias prefixed source",
firstSession: "",
firstSource: "a\x00b",
secondSession: "\x00a",
secondSource: "b",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
first := CanonicalMessageID(tc.firstSession, tc.firstSource)
second := CanonicalMessageID(tc.secondSession, tc.secondSource)
if first == second {
t.Fatalf("CanonicalMessageID(%q, %q) = CanonicalMessageID(%q, %q) = %q",
tc.firstSession, tc.firstSource, tc.secondSession, tc.secondSource, first)
}
})
}
}
func TestSQLiteSessionStoreRecognisesLegacyCanonicalIDOnRedelivery(t *testing.T) {
t.Parallel()
store := newTestSQLiteStore(t)
ctx := t.Context()
const (
sessionID = "a\x00b"
sourceID = "source"
)
legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
if _, err := sqliteStoreDB(t, store).ExecContext(ctx, `
INSERT INTO messages (message_id, session_id, source_id, sender, text, ts)
VALUES (?, ?, ?, 'alice', 'original', ?)`,
legacyID, sessionID, sourceID, time.Now().Add(-time.Minute).UnixMilli()); err != nil {
t.Fatalf("seed legacy message: %v", err)
}
if err := store.SaveMessage(ctx, sessionID, Message{
MessageID: CanonicalMessageID(sessionID, sourceID),
SourceID: sourceID, From: "alice", Text: "redelivered",
}); err != nil {
t.Fatalf("SaveMessage(redelivery): %v", err)
}
messages, err := store.RecentMessages(ctx, sessionID, 10)
if err != nil {
t.Fatalf("RecentMessages: %v", err)
}
if len(messages) != 1 {
t.Fatalf("redelivery produced %d messages, want 1", len(messages))
}
if messages[0].MessageID != legacyID || messages[0].Text != "original" {
t.Fatalf("legacy message changed on redelivery: %#v", messages[0])
}
}
func TestSQLiteSessionStoreFindPriorReplyRecognisesLegacyCanonicalID(t *testing.T) {
t.Parallel()
store := newTestSQLiteStore(t)
ctx := t.Context()
const (
sessionID = "a\x00b"
sourceID = "source"
identity = "archie"
)
legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
inputAt := time.Now().Add(-time.Minute).UnixMilli()
statements := []struct {
id string
source string
sender string
text string
at int64
}{
{id: legacyID, source: sourceID, sender: "alice", text: "question", at: inputAt},
{id: "reply", sender: identity, text: "answer", at: inputAt + 1},
}
for _, row := range statements {
if _, err := sqliteStoreDB(t, store).ExecContext(ctx, `
INSERT INTO messages (message_id, session_id, source_id, sender, text, ts)
VALUES (?, ?, ?, ?, ?, ?)`,
row.id, sessionID, row.source, row.sender, row.text, row.at); err != nil {
t.Fatalf("seed %s: %v", row.id, err)
}
}
replayStore, ok := store.(TurnReplayStore)
if !ok {
t.Fatalf("store is %T, want TurnReplayStore", store)
}
got, err := replayStore.FindPriorReply(ctx, sessionID, sourceID, identity)
if err != nil {
t.Fatalf("FindPriorReply: %v", err)
}
if got != "answer" {
t.Fatalf("FindPriorReply = %q, want legacy reply", got)
}
}
func TestSQLiteSessionStoreCapsFutureMessageTimestamp(t *testing.T) {
t.Parallel()
store := newTestSQLiteStore(t)
ctx := t.Context()
now := time.Now().UTC()
if err := store.SaveMessage(ctx, "sess", Message{
From: "alice", Text: "future", At: now.Add(24 * time.Hour),
}); err != nil {
t.Fatalf("SaveMessage(future): %v", err)
}
if err := store.SaveMessage(ctx, "sess", Message{
From: "alice", Text: "current", At: now,
}); err != nil {
t.Fatalf("SaveMessage(current): %v", err)
}
messages, err := store.RecentMessages(ctx, "sess", 10)
if err != nil {
t.Fatalf("RecentMessages: %v", err)
}
if len(messages) != 2 {
t.Fatalf("RecentMessages returned %d messages, want 2", len(messages))
}
if messages[0].At.After(time.Now().UTC().Add(time.Minute)) {
t.Fatalf("future timestamp was persisted as %s", messages[0].At)
}
if !messages[1].At.After(messages[0].At) {
t.Fatalf("current message timestamp %s does not follow capped future timestamp %s",
messages[1].At, messages[0].At)
}
}
func TestSQLiteSessionStoreRepairsPersistedFutureTimestampsOnReopen(t *testing.T) {
t.Parallel()
ctx := t.Context()
path := filepath.Join(t.TempDir(), "future.db")
store, err := OpenSQLiteSessionStore(path)
if err != nil {
t.Fatalf("OpenSQLiteSessionStore: %v", err)
}
now := time.Now().UTC()
rows := []struct {
id string
text string
at time.Time
}{
{id: "past", text: "past", at: now.Add(-time.Minute)},
{id: "future-1", text: "future-1", at: now.Add(24 * time.Hour)},
{id: "future-2", text: "future-2", at: now.Add(25 * time.Hour)},
}
for _, row := range rows {
if _, err := sqliteStoreDB(t, store).ExecContext(ctx, `
INSERT INTO messages (message_id, session_id, sender, text, ts)
VALUES (?, 'sess', 'alice', ?, ?)`, row.id, row.text, row.at.UnixMilli()); err != nil {
t.Fatalf("seed %s: %v", row.id, err)
}
}
if err := store.Close(); err != nil {
t.Fatalf("Close seeded store: %v", err)
}
reopened, err := OpenSQLiteSessionStore(path)
if err != nil {
t.Fatalf("reopen store: %v", err)
}
t.Cleanup(func() { _ = reopened.Close() })
if err := reopened.SaveMessage(ctx, "sess", Message{
From: "alice", Text: "current", At: time.Now().UTC(),
}); err != nil {
t.Fatalf("SaveMessage(current): %v", err)
}
messages, err := reopened.RecentMessages(ctx, "sess", 10)
if err != nil {
t.Fatalf("RecentMessages: %v", err)
}
want := []string{"past", "future-1", "future-2", "current"}
if got := texts(messages); !equal(got, want) {
t.Fatalf("history = %v, want %v", got, want)
}
for i := 1; i < len(messages); i++ {
if !messages[i].At.After(messages[i-1].At) {
t.Fatalf("timestamp %d (%s) does not follow %d (%s)",
i, messages[i].At, i-1, messages[i-1].At)
}
}
if messages[len(messages)-1].At.After(time.Now().UTC().Add(time.Second)) {
t.Fatalf("repaired history remains pinned in the future: last timestamp %s",
messages[len(messages)-1].At)
}
}
func TestSQLiteMessageTimestampRoundTrips(t *testing.T) {
st := newTestSQLiteStore(t)
want := sqliteBase().Add(90*time.Minute + 123*time.Millisecond)
+26 -1
View File
@@ -146,7 +146,10 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, onDelta func(string))
if err != nil {
return "", fmt.Errorf("resolve chat session: %w", err)
}
turnID := CanonicalTurnID(sessionID, msg.SourceID)
turnID, err := r.canonicalTurnID(ctx, sessionID, msg.SourceID)
if err != nil {
return "", fmt.Errorf("resolve canonical turn ID: %w", err)
}
if turnID == "" {
turnID = NewTurnID()
}
@@ -303,6 +306,28 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, onDelta func(string))
return text, nil
}
// canonicalTurnID preserves a matching identifier written by the legacy
// separator encoding. A legacy collision belonging to a different
// session/source pair is ignored in favour of the injective identifier.
func (r *TurnRunner) canonicalTurnID(ctx context.Context, sessionID, sourceID string) (string, error) {
current := CanonicalTurnID(sessionID, sourceID)
if current == "" {
return "", nil
}
legacy := legacyCanonicalTurnID(sessionID, sourceID)
if current == legacy {
return current, nil
}
turn, ok, err := r.Ledger.GetTurn(ctx, legacy)
if err != nil {
return "", err
}
if ok && turn.SessionID == sessionID && turn.SourceID == sourceID {
return legacy, nil
}
return current, nil
}
func (r *TurnRunner) failTurn(ctx context.Context, turn TurnRecord, cause error) error {
if errors.Is(cause, context.Canceled) {
turn.Status = TurnStatusCancelled
+11
View File
@@ -3,6 +3,7 @@ package gateway
import (
"context"
"errors"
"strings"
"time"
"github.com/google/uuid"
@@ -80,9 +81,19 @@ func assistantMessageIDForTurn(turnID string) string {
return uuid.NewSHA1(uuid.NameSpaceURL, []byte("archie-assistant-turn:\x00"+turnID)).String()
}
var turnIDNamespaceV2 = uuid.MustParse("59ca315e-3b2d-4a2f-bd46-93e0ebf30788")
func CanonicalTurnID(sessionID, sourceID string) string {
if sourceID == "" {
return ""
}
legacy := legacyCanonicalTurnID(sessionID, sourceID)
if strings.ContainsRune(sessionID, '\x00') || strings.ContainsRune(sourceID, '\x00') {
return uuid.NewSHA1(turnIDNamespaceV2, encodeCanonicalPair(sessionID, sourceID)).String()
}
return legacy
}
func legacyCanonicalTurnID(sessionID, sourceID string) string {
return uuid.NewSHA1(uuid.NameSpaceURL, []byte("archie-turn:\x00"+sessionID+"\x00"+sourceID)).String()
}
+80 -1
View File
@@ -203,7 +203,10 @@ func TestSQLiteTurnLedgerSurvivesReopen(t *testing.T) {
if err != nil {
t.Fatalf("OpenSQLiteSessionStore() error = %v", err)
}
ledger := store.(TurnLedger)
ledger, ok := store.(TurnLedger)
if !ok {
t.Fatalf("store is %T, want TurnLedger", store)
}
turn := TurnRecord{
TurnID: CanonicalTurnID("session-reopen", "source-reopen"),
SessionID: "session-reopen",
@@ -253,3 +256,79 @@ func TestCanonicalTurnIDIsStableAndSessionScoped(t *testing.T) {
t.Fatalf("empty source ID produced %q, want empty", got)
}
}
func TestCanonicalTurnIDComponentsAreInjectivelyEncoded(t *testing.T) {
t.Parallel()
first := CanonicalTurnID("a\x00b", "c")
second := CanonicalTurnID("a", "b\x00c")
if first == second {
t.Fatalf("ambiguous component pairs shared turn ID %q", first)
}
}
func TestTurnRunnerCanonicalTurnIDPreservesMatchingLegacyRecord(t *testing.T) {
t.Parallel()
store := newTestSQLiteStore(t)
ledger, ok := store.(TurnLedger)
if !ok {
t.Fatalf("store is %T, want TurnLedger", store)
}
ctx := t.Context()
const (
sessionID = "a\x00b"
sourceID = "c"
)
legacyID := legacyCanonicalTurnID(sessionID, sourceID)
if _, claim, err := ledger.ClaimTurn(ctx, TurnRecord{
TurnID: legacyID, SessionID: sessionID, SourceID: sourceID,
Status: TurnStatusAccepted, OwnerID: "old-process",
}); err != nil || claim != TurnClaimOwned {
t.Fatalf("seed legacy turn = %q, %v", claim, err)
}
runner := NewTurnRunner(TurnRunnerConfig{Ledger: ledger})
got, err := runner.canonicalTurnID(ctx, sessionID, sourceID)
if err != nil {
t.Fatalf("canonicalTurnID: %v", err)
}
if got != legacyID {
t.Fatalf("canonicalTurnID = %q, want matching legacy ID %q", got, legacyID)
}
}
func TestTurnRunnerCanonicalTurnIDRejectsCollidingLegacyRecord(t *testing.T) {
t.Parallel()
store := newTestSQLiteStore(t)
ledger, ok := store.(TurnLedger)
if !ok {
t.Fatalf("store is %T, want TurnLedger", store)
}
ctx := t.Context()
const (
firstSession = "a\x00b"
firstSource = "c"
secondSession = "a"
secondSource = "b\x00c"
)
collidingLegacyID := legacyCanonicalTurnID(firstSession, firstSource)
if _, claim, err := ledger.ClaimTurn(ctx, TurnRecord{
TurnID: collidingLegacyID, SessionID: firstSession, SourceID: firstSource,
Status: TurnStatusAccepted, OwnerID: "first-process",
}); err != nil || claim != TurnClaimOwned {
t.Fatalf("seed colliding turn = %q, %v", claim, err)
}
runner := NewTurnRunner(TurnRunnerConfig{Ledger: ledger})
got, err := runner.canonicalTurnID(ctx, secondSession, secondSource)
if err != nil {
t.Fatalf("canonicalTurnID: %v", err)
}
want := CanonicalTurnID(secondSession, secondSource)
if got != want || got == collidingLegacyID {
t.Fatalf("canonicalTurnID = %q, want isolated ID %q (legacy collision %q)",
got, want, collidingLegacyID)
}
}
+38 -15
View File
@@ -76,16 +76,14 @@ func NewTurns(log *slog.Logger) *Turns {
//
// Submit never blocks and never rejects: messages queue.
func (t *Turns) Submit(ctx context.Context, session string, run func(context.Context)) {
turnCtx, cancel := context.WithCancel(ctx)
t.mu.Lock()
l, ok := t.lanes[session]
if !ok {
l = &lane{}
t.lanes[session] = l
}
t.mu.Unlock()
turnCtx, cancel := context.WithCancel(ctx)
l.mu.Lock()
l.pending = append(l.pending, queued{run: func() { run(turnCtx) }, cancel: cancel})
depth := len(l.pending)
@@ -94,9 +92,10 @@ func (t *Turns) Submit(ctx context.Context, session string, run func(context.Con
l.working = true
}
l.mu.Unlock()
t.mu.Unlock()
if start {
go l.serve(t.log.With("session", session))
go t.serve(session, l)
}
if depth > 1 {
t.log.Info("chat turn queued", "session", session, "queue_depth", depth)
@@ -154,7 +153,8 @@ func (t *Turns) Queued(session string) int {
return len(l.pending)
}
// lane returns the session's lane, or nil if it has never been used.
// lane returns the session's active lane, or nil if it has never been used or
// its completed lane has already retired.
func (t *Turns) lane(session string) *lane {
t.mu.Lock()
defer t.mu.Unlock()
@@ -164,19 +164,45 @@ func (t *Turns) lane(session string) *lane {
// serve drains the lane's backlog one turn at a time and exits once it is
// empty. Submit starts a fresh worker when more arrives, so an idle session
// costs no goroutine.
func (l *lane) serve(log *slog.Logger) {
func (t *Turns) serve(session string, l *lane) {
log := t.log.With("session", session)
for {
next, ok := l.begin()
if !ok {
return
if t.retire(session, l) {
return
}
continue
}
l.runOne(next, log)
}
}
// retire removes an idle lane while holding the same two locks Submit uses to
// find and append to it. Without that joint critical section, Submit could find
// the old lane immediately before cleanup deleted it and strand the new turn on
// a worker that had already exited.
func (t *Turns) retire(session string, l *lane) bool {
t.mu.Lock()
defer t.mu.Unlock()
current, ok := t.lanes[session]
if !ok || current != l {
return true
}
l.mu.Lock()
defer l.mu.Unlock()
if len(l.pending) != 0 {
return false
}
l.working = false
delete(t.lanes, session)
return true
}
// begin pops the oldest queued turn and publishes its cancel in the same
// critical section, reporting false when the backlog is empty -- at which
// point it also retires the worker.
// critical section. It reports false when the backlog is empty; the owning
// Turns then retires the worker and map entry under both locks.
//
// The pop and the publish must happen together. Were the turn popped first
// and its cancel published after, a Stop landing in between would see an
@@ -187,11 +213,8 @@ func (l *lane) begin() (queued, bool) {
defer l.mu.Unlock()
if len(l.pending) == 0 {
// Retire under the same lock Submit uses to decide whether a
// worker is live, so a turn arriving now either starts a new
// worker or is picked up before this one retires -- never
// neither, which would strand the backlog forever.
l.working = false
// The owning Turns retires the lane under both the map and lane
// locks. Changing working here would race a concurrent Submit.
return queued{}, false
}
+52
View File
@@ -0,0 +1,52 @@
package gateway
import (
"context"
"log/slog"
"testing"
"time"
)
func TestTurnsRetiresIdleSessionLanes(t *testing.T) {
t.Parallel()
turns := NewTurns(slog.New(slog.DiscardHandler))
const sessions = 100
done := make(chan struct{}, sessions)
for i := range sessions {
turns.Submit(context.Background(), string(rune(i)), func(context.Context) {
done <- struct{}{}
})
}
for range sessions {
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("submitted turn did not finish")
}
}
deadline := time.Now().Add(2 * time.Second)
for {
turns.mu.Lock()
remaining := len(turns.lanes)
turns.mu.Unlock()
if remaining == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("%d idle session lanes remain allocated", remaining)
}
time.Sleep(time.Millisecond)
}
// A retired session must remain reusable; cleanup cannot strand a turn
// between looking up the old lane and creating its replacement.
reused := make(chan struct{})
turns.Submit(context.Background(), string(rune(0)), func(context.Context) { close(reused) })
select {
case <-reused:
case <-time.After(2 * time.Second):
t.Fatal("retired session did not accept a new turn")
}
}