diff --git a/internal/gateway/redelivery_test.go b/internal/gateway/redelivery_test.go index 0bec633e9..662713477 100644 --- a/internal/gateway/redelivery_test.go +++ b/internal/gateway/redelivery_test.go @@ -3,8 +3,28 @@ package gateway import ( "testing" "time" + + "github.com/google/uuid" ) +func TestPriorReplyRecognisesLegacyCanonicalMessageID(t *testing.T) { + t.Parallel() + + const ( + sessionID = "a\x00b" + sourceID = "source" + identity = "archie" + ) + legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String() + history := []Message{ + {MessageID: legacyID, SourceID: sourceID, From: "alice", Text: "question"}, + {From: identity, Text: "answer"}, + } + if got := PriorReply(history, sessionID, sourceID, identity); got != "answer" { + t.Fatalf("PriorReply = %q, want legacy reply", got) + } +} + // Making the store idempotent only stopped the duplicate *record*. The chat // path saved the inbound message, ignored the fact that nothing changed, and // went on to call the model again -- a second billed turn, any tool side diff --git a/internal/gateway/session_commands_test.go b/internal/gateway/session_commands_test.go index 91e691b2b..bc8ba4e19 100644 --- a/internal/gateway/session_commands_test.go +++ b/internal/gateway/session_commands_test.go @@ -462,6 +462,36 @@ func TestRouteUndoRemovesMultiple(t *testing.T) { } } +func TestRouteUndoReportsActualPartialRemoval(t *testing.T) { + r := newTestRouter("telegram") + ctx := t.Context() + const sessionID = "test-undo-partial" + sc := SessionContext{ + SessionID: sessionID, + Source: SessionSource{Platform: "telegram", BotUser: "archie", ChannelID: "chat-partial"}, + CreatedAt: time.Now(), LastActiveAt: time.Now(), + } + if err := r.sessionTracker.sessions.Save(ctx, sc); err != nil { + t.Fatalf("Save session: %v", err) + } + for i := range 6 { + if err := r.sessionTracker.sessions.SaveMessage(ctx, sessionID, Message{ + From: "alice", Text: fmt.Sprintf("msg%d", i), + }); err != nil { + t.Fatalf("SaveMessage(%d): %v", i, err) + } + } + r.sessionTracker.setActive("chat-partial", "", sessionID) + + reply, err := r.Route(ctx, Message{Text: "/undo 10", ChannelID: "chat-partial"}) + if err != nil { + t.Fatalf("Route: %v", err) + } + if !strings.Contains(reply, "Removed 6 messages") { + t.Fatalf("reply = %q, want actual removal count", reply) + } +} + func TestRouteUndoNoMessages(t *testing.T) { r := newTestRouter("telegram") diff --git a/internal/gateway/session_store.go b/internal/gateway/session_store.go index 77bf8dec5..791ab5b92 100644 --- a/internal/gateway/session_store.go +++ b/internal/gateway/session_store.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "encoding/binary" "strings" "time" @@ -101,16 +102,42 @@ func sessionRecency(sc SessionContext) time.Time { // messageIDNamespace scopes deterministic canonical message identifiers. var messageIDNamespace = uuid.MustParse("6f8d2b1e-9c4a-4f37-8a56-1d0e7b3c9a42") +// messageIDNamespaceV2 isolates the injective encoding used for inputs that +// contain the legacy separator. Ordinary platform/session IDs keep their +// existing canonical IDs, so an upgrade cannot duplicate redelivered messages. +var messageIDNamespaceV2 = uuid.MustParse("e6ac7868-cf49-4e65-a5a1-a1549fd12f67") + func newMessageID(sessionID, sourceID string) string { if sourceID == "" { return uuid.NewString() } - return uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String() + current, _ := canonicalMessageIDs(sessionID, sourceID) + return current +} + +func canonicalMessageIDs(sessionID, sourceID string) (current, legacy string) { + legacy = uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String() + if strings.ContainsRune(sessionID, '\x00') || strings.ContainsRune(sourceID, '\x00') { + return uuid.NewSHA1(messageIDNamespaceV2, encodeCanonicalPair(sessionID, sourceID)).String(), legacy + } + return legacy, "" +} + +func encodeCanonicalPair(first, second string) []byte { + encoded := make([]byte, 8+len(first)+len(second)) + binary.BigEndian.PutUint64(encoded, uint64(len(first))) + copy(encoded[8:], first) + copy(encoded[8+len(first):], second) + return encoded } func stamp(msg Message) time.Time { - if msg.At.IsZero() { - return time.Now().UTC() + now := time.Now().UTC() + if msg.At.IsZero() || msg.At.After(now) { + // At orders the canonical conversation; it is not an authoritative + // copy of a platform clock. Persisting a future platform timestamp + // would force every later message beyond it to preserve monotonicity. + return now } return msg.At.UTC() } @@ -125,12 +152,12 @@ func CanonicalMessageID(sessionID, sourceID string) string { // PriorReply returns the reply already produced for an upstream message. func PriorReply(history []Message, sessionID, sourceID, identity string) string { - id := CanonicalMessageID(sessionID, sourceID) - if id == "" { + if sourceID == "" { return "" } + id, legacyID := canonicalMessageIDs(sessionID, sourceID) for i, message := range history { - if message.MessageID != id { + if message.MessageID != id && message.MessageID != legacyID { continue } if i+1 < len(history) && history[i+1].From == identity && diff --git a/internal/gateway/session_store_sqlite.go b/internal/gateway/session_store_sqlite.go index df54ca9b5..e75fafe29 100644 --- a/internal/gateway/session_store_sqlite.go +++ b/internal/gateway/session_store_sqlite.go @@ -151,9 +151,43 @@ func openSQLiteSessionStore(dsn string) (SessionStore, error) { if _, err := db.ExecContext(ctx, sqliteSessionSchema); err != nil { return nil, errors.Join(fmt.Errorf("sessionstore: init schema: %w", err), db.Close()) } + if err := repairFutureMessageTimestamps(ctx, db, time.Now().UTC()); err != nil { + return nil, errors.Join(err, db.Close()) + } return &sqliteSessionStore{db: db}, nil } +// repairFutureMessageTimestamps heals sessions poisoned by older versions +// that accepted a platform timestamp arbitrarily far ahead of the daemon's +// clock. Future rows retain their order and are rebased immediately after the +// session's last non-future row. With no earlier row, they end at now. A dense +// history can finish a few milliseconds beyond now, but never preserves the +// original hours- or days-long skew. +func repairFutureMessageTimestamps(ctx context.Context, db *sql.DB, now time.Time) error { + cutoff := now.UnixMilli() + _, err := db.ExecContext(ctx, ` + WITH future AS MATERIALIZED ( + SELECT m.id, + COALESCE( + (SELECT MAX(p.ts) FROM messages p + WHERE p.session_id = m.session_id AND p.ts <= ?), + ? - COUNT(*) OVER (PARTITION BY m.session_id) + ) + ROW_NUMBER() OVER ( + PARTITION BY m.session_id ORDER BY m.ts, m.id + ) AS repaired_ts + FROM messages m + WHERE m.ts > ? + ) + UPDATE messages + SET ts = (SELECT repaired_ts FROM future WHERE future.id = messages.id) + WHERE id IN (SELECT id FROM future) AND ts > ?`, + cutoff, cutoff, cutoff, cutoff) + if err != nil { + return fmt.Errorf("sessionstore: repair future message timestamps: %w", err) + } + return nil +} + func (s *sqliteSessionStore) Close() error { return s.db.Close() } @@ -579,8 +613,34 @@ func saveMessage(ctx context.Context, ex execer, sessionID string, msg Message) // placing records deliberately. func saveMessageAt(ctx context.Context, ex execer, sessionID string, msg Message, clamp bool) (string, error) { id := msg.MessageID - if id == "" { - id = newMessageID(sessionID, msg.SourceID) + legacyID := "" + if msg.SourceID == "" { + if id == "" { + id = newMessageID(sessionID, "") + } + } else { + currentID, compatibleLegacyID := canonicalMessageIDs(sessionID, msg.SourceID) + if id == "" { + id = currentID + } + // TurnRunner supplies the current canonical ID before calling the + // store. Still check the compatible legacy identity: a crash can + // leave the old message durable before its turn record names it. + if id == currentID { + legacyID = compatibleLegacyID + } + } + if legacyID != "" { + var existingID string + err := ex.QueryRowContext(ctx, + `SELECT message_id FROM messages WHERE session_id = ? AND message_id = ?`, + sessionID, legacyID).Scan(&existingID) + switch { + case err == nil: + return existingID, nil + case !errors.Is(err, sql.ErrNoRows): + return "", fmt.Errorf("sessionstore: read legacy message identity: %w", err) + } } at := stamp(msg) if clamp { @@ -710,10 +770,13 @@ func (s *sqliteSessionStore) FindPriorReply(ctx context.Context, sessionID, sour s.mu.Lock() defer s.mu.Unlock() + currentID, legacyID := canonicalMessageIDs(sessionID, sourceID) var ts int64 err := s.db.QueryRowContext(ctx, ` - SELECT ts FROM messages WHERE session_id = ? AND message_id = ?`, - sessionID, CanonicalMessageID(sessionID, sourceID)).Scan(&ts) + SELECT ts FROM messages + WHERE session_id = ? AND message_id IN (?, ?) + ORDER BY CASE WHEN message_id = ? THEN 0 ELSE 1 END + LIMIT 1`, sessionID, currentID, legacyID, legacyID).Scan(&ts) if errors.Is(err, sql.ErrNoRows) { return "", nil } diff --git a/internal/gateway/session_store_sqlite_test.go b/internal/gateway/session_store_sqlite_test.go index ac9266e44..68d4ef31d 100644 --- a/internal/gateway/session_store_sqlite_test.go +++ b/internal/gateway/session_store_sqlite_test.go @@ -9,6 +9,8 @@ import ( "sync" "testing" "time" + + "github.com/google/uuid" ) func newTestSQLiteStore(t *testing.T) SessionStore { @@ -93,6 +95,217 @@ func TestMessageIDsAreUnique(t *testing.T) { } } +func TestCanonicalMessageIDComponentsAreInjectivelyEncoded(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + firstSession string + firstSource string + secondSession string + secondSource string + }{ + { + name: "separator can occur in session", + firstSession: "a\x00b", + firstSource: "c", + secondSession: "a", + secondSource: "b\x00c", + }, + { + name: "empty session does not alias prefixed source", + firstSession: "", + firstSource: "a\x00b", + secondSession: "\x00a", + secondSource: "b", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + first := CanonicalMessageID(tc.firstSession, tc.firstSource) + second := CanonicalMessageID(tc.secondSession, tc.secondSource) + if first == second { + t.Fatalf("CanonicalMessageID(%q, %q) = CanonicalMessageID(%q, %q) = %q", + tc.firstSession, tc.firstSource, tc.secondSession, tc.secondSource, first) + } + }) + } +} + +func TestSQLiteSessionStoreRecognisesLegacyCanonicalIDOnRedelivery(t *testing.T) { + t.Parallel() + + store := newTestSQLiteStore(t) + ctx := t.Context() + const ( + sessionID = "a\x00b" + sourceID = "source" + ) + legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String() + if _, err := sqliteStoreDB(t, store).ExecContext(ctx, ` + INSERT INTO messages (message_id, session_id, source_id, sender, text, ts) + VALUES (?, ?, ?, 'alice', 'original', ?)`, + legacyID, sessionID, sourceID, time.Now().Add(-time.Minute).UnixMilli()); err != nil { + t.Fatalf("seed legacy message: %v", err) + } + + if err := store.SaveMessage(ctx, sessionID, Message{ + MessageID: CanonicalMessageID(sessionID, sourceID), + SourceID: sourceID, From: "alice", Text: "redelivered", + }); err != nil { + t.Fatalf("SaveMessage(redelivery): %v", err) + } + messages, err := store.RecentMessages(ctx, sessionID, 10) + if err != nil { + t.Fatalf("RecentMessages: %v", err) + } + if len(messages) != 1 { + t.Fatalf("redelivery produced %d messages, want 1", len(messages)) + } + if messages[0].MessageID != legacyID || messages[0].Text != "original" { + t.Fatalf("legacy message changed on redelivery: %#v", messages[0]) + } +} + +func TestSQLiteSessionStoreFindPriorReplyRecognisesLegacyCanonicalID(t *testing.T) { + t.Parallel() + + store := newTestSQLiteStore(t) + ctx := t.Context() + const ( + sessionID = "a\x00b" + sourceID = "source" + identity = "archie" + ) + legacyID := uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String() + inputAt := time.Now().Add(-time.Minute).UnixMilli() + statements := []struct { + id string + source string + sender string + text string + at int64 + }{ + {id: legacyID, source: sourceID, sender: "alice", text: "question", at: inputAt}, + {id: "reply", sender: identity, text: "answer", at: inputAt + 1}, + } + for _, row := range statements { + if _, err := sqliteStoreDB(t, store).ExecContext(ctx, ` + INSERT INTO messages (message_id, session_id, source_id, sender, text, ts) + VALUES (?, ?, ?, ?, ?, ?)`, + row.id, sessionID, row.source, row.sender, row.text, row.at); err != nil { + t.Fatalf("seed %s: %v", row.id, err) + } + } + + replayStore, ok := store.(TurnReplayStore) + if !ok { + t.Fatalf("store is %T, want TurnReplayStore", store) + } + got, err := replayStore.FindPriorReply(ctx, sessionID, sourceID, identity) + if err != nil { + t.Fatalf("FindPriorReply: %v", err) + } + if got != "answer" { + t.Fatalf("FindPriorReply = %q, want legacy reply", got) + } +} + +func TestSQLiteSessionStoreCapsFutureMessageTimestamp(t *testing.T) { + t.Parallel() + + store := newTestSQLiteStore(t) + ctx := t.Context() + now := time.Now().UTC() + if err := store.SaveMessage(ctx, "sess", Message{ + From: "alice", Text: "future", At: now.Add(24 * time.Hour), + }); err != nil { + t.Fatalf("SaveMessage(future): %v", err) + } + if err := store.SaveMessage(ctx, "sess", Message{ + From: "alice", Text: "current", At: now, + }); err != nil { + t.Fatalf("SaveMessage(current): %v", err) + } + + messages, err := store.RecentMessages(ctx, "sess", 10) + if err != nil { + t.Fatalf("RecentMessages: %v", err) + } + if len(messages) != 2 { + t.Fatalf("RecentMessages returned %d messages, want 2", len(messages)) + } + if messages[0].At.After(time.Now().UTC().Add(time.Minute)) { + t.Fatalf("future timestamp was persisted as %s", messages[0].At) + } + if !messages[1].At.After(messages[0].At) { + t.Fatalf("current message timestamp %s does not follow capped future timestamp %s", + messages[1].At, messages[0].At) + } +} + +func TestSQLiteSessionStoreRepairsPersistedFutureTimestampsOnReopen(t *testing.T) { + t.Parallel() + + ctx := t.Context() + path := filepath.Join(t.TempDir(), "future.db") + store, err := OpenSQLiteSessionStore(path) + if err != nil { + t.Fatalf("OpenSQLiteSessionStore: %v", err) + } + now := time.Now().UTC() + rows := []struct { + id string + text string + at time.Time + }{ + {id: "past", text: "past", at: now.Add(-time.Minute)}, + {id: "future-1", text: "future-1", at: now.Add(24 * time.Hour)}, + {id: "future-2", text: "future-2", at: now.Add(25 * time.Hour)}, + } + for _, row := range rows { + if _, err := sqliteStoreDB(t, store).ExecContext(ctx, ` + INSERT INTO messages (message_id, session_id, sender, text, ts) + VALUES (?, 'sess', 'alice', ?, ?)`, row.id, row.text, row.at.UnixMilli()); err != nil { + t.Fatalf("seed %s: %v", row.id, err) + } + } + if err := store.Close(); err != nil { + t.Fatalf("Close seeded store: %v", err) + } + + reopened, err := OpenSQLiteSessionStore(path) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + t.Cleanup(func() { _ = reopened.Close() }) + if err := reopened.SaveMessage(ctx, "sess", Message{ + From: "alice", Text: "current", At: time.Now().UTC(), + }); err != nil { + t.Fatalf("SaveMessage(current): %v", err) + } + + messages, err := reopened.RecentMessages(ctx, "sess", 10) + if err != nil { + t.Fatalf("RecentMessages: %v", err) + } + want := []string{"past", "future-1", "future-2", "current"} + if got := texts(messages); !equal(got, want) { + t.Fatalf("history = %v, want %v", got, want) + } + for i := 1; i < len(messages); i++ { + if !messages[i].At.After(messages[i-1].At) { + t.Fatalf("timestamp %d (%s) does not follow %d (%s)", + i, messages[i].At, i-1, messages[i-1].At) + } + } + if messages[len(messages)-1].At.After(time.Now().UTC().Add(time.Second)) { + t.Fatalf("repaired history remains pinned in the future: last timestamp %s", + messages[len(messages)-1].At) + } +} + func TestSQLiteMessageTimestampRoundTrips(t *testing.T) { st := newTestSQLiteStore(t) want := sqliteBase().Add(90*time.Minute + 123*time.Millisecond) diff --git a/internal/gateway/turn.go b/internal/gateway/turn.go index 3e508de5b..0b026ff19 100644 --- a/internal/gateway/turn.go +++ b/internal/gateway/turn.go @@ -146,7 +146,10 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, onDelta func(string)) if err != nil { return "", fmt.Errorf("resolve chat session: %w", err) } - turnID := CanonicalTurnID(sessionID, msg.SourceID) + turnID, err := r.canonicalTurnID(ctx, sessionID, msg.SourceID) + if err != nil { + return "", fmt.Errorf("resolve canonical turn ID: %w", err) + } if turnID == "" { turnID = NewTurnID() } @@ -303,6 +306,28 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, onDelta func(string)) return text, nil } +// canonicalTurnID preserves a matching identifier written by the legacy +// separator encoding. A legacy collision belonging to a different +// session/source pair is ignored in favour of the injective identifier. +func (r *TurnRunner) canonicalTurnID(ctx context.Context, sessionID, sourceID string) (string, error) { + current := CanonicalTurnID(sessionID, sourceID) + if current == "" { + return "", nil + } + legacy := legacyCanonicalTurnID(sessionID, sourceID) + if current == legacy { + return current, nil + } + turn, ok, err := r.Ledger.GetTurn(ctx, legacy) + if err != nil { + return "", err + } + if ok && turn.SessionID == sessionID && turn.SourceID == sourceID { + return legacy, nil + } + return current, nil +} + func (r *TurnRunner) failTurn(ctx context.Context, turn TurnRecord, cause error) error { if errors.Is(cause, context.Canceled) { turn.Status = TurnStatusCancelled diff --git a/internal/gateway/turn_ledger.go b/internal/gateway/turn_ledger.go index efe281278..04ccfef68 100644 --- a/internal/gateway/turn_ledger.go +++ b/internal/gateway/turn_ledger.go @@ -3,6 +3,7 @@ package gateway import ( "context" "errors" + "strings" "time" "github.com/google/uuid" @@ -80,9 +81,19 @@ func assistantMessageIDForTurn(turnID string) string { return uuid.NewSHA1(uuid.NameSpaceURL, []byte("archie-assistant-turn:\x00"+turnID)).String() } +var turnIDNamespaceV2 = uuid.MustParse("59ca315e-3b2d-4a2f-bd46-93e0ebf30788") + func CanonicalTurnID(sessionID, sourceID string) string { if sourceID == "" { return "" } + legacy := legacyCanonicalTurnID(sessionID, sourceID) + if strings.ContainsRune(sessionID, '\x00') || strings.ContainsRune(sourceID, '\x00') { + return uuid.NewSHA1(turnIDNamespaceV2, encodeCanonicalPair(sessionID, sourceID)).String() + } + return legacy +} + +func legacyCanonicalTurnID(sessionID, sourceID string) string { return uuid.NewSHA1(uuid.NameSpaceURL, []byte("archie-turn:\x00"+sessionID+"\x00"+sourceID)).String() } diff --git a/internal/gateway/turn_ledger_test.go b/internal/gateway/turn_ledger_test.go index 753f3aa62..e997e3d4a 100644 --- a/internal/gateway/turn_ledger_test.go +++ b/internal/gateway/turn_ledger_test.go @@ -203,7 +203,10 @@ func TestSQLiteTurnLedgerSurvivesReopen(t *testing.T) { if err != nil { t.Fatalf("OpenSQLiteSessionStore() error = %v", err) } - ledger := store.(TurnLedger) + ledger, ok := store.(TurnLedger) + if !ok { + t.Fatalf("store is %T, want TurnLedger", store) + } turn := TurnRecord{ TurnID: CanonicalTurnID("session-reopen", "source-reopen"), SessionID: "session-reopen", @@ -253,3 +256,79 @@ func TestCanonicalTurnIDIsStableAndSessionScoped(t *testing.T) { t.Fatalf("empty source ID produced %q, want empty", got) } } + +func TestCanonicalTurnIDComponentsAreInjectivelyEncoded(t *testing.T) { + t.Parallel() + + first := CanonicalTurnID("a\x00b", "c") + second := CanonicalTurnID("a", "b\x00c") + if first == second { + t.Fatalf("ambiguous component pairs shared turn ID %q", first) + } +} + +func TestTurnRunnerCanonicalTurnIDPreservesMatchingLegacyRecord(t *testing.T) { + t.Parallel() + + store := newTestSQLiteStore(t) + ledger, ok := store.(TurnLedger) + if !ok { + t.Fatalf("store is %T, want TurnLedger", store) + } + ctx := t.Context() + const ( + sessionID = "a\x00b" + sourceID = "c" + ) + legacyID := legacyCanonicalTurnID(sessionID, sourceID) + if _, claim, err := ledger.ClaimTurn(ctx, TurnRecord{ + TurnID: legacyID, SessionID: sessionID, SourceID: sourceID, + Status: TurnStatusAccepted, OwnerID: "old-process", + }); err != nil || claim != TurnClaimOwned { + t.Fatalf("seed legacy turn = %q, %v", claim, err) + } + + runner := NewTurnRunner(TurnRunnerConfig{Ledger: ledger}) + got, err := runner.canonicalTurnID(ctx, sessionID, sourceID) + if err != nil { + t.Fatalf("canonicalTurnID: %v", err) + } + if got != legacyID { + t.Fatalf("canonicalTurnID = %q, want matching legacy ID %q", got, legacyID) + } +} + +func TestTurnRunnerCanonicalTurnIDRejectsCollidingLegacyRecord(t *testing.T) { + t.Parallel() + + store := newTestSQLiteStore(t) + ledger, ok := store.(TurnLedger) + if !ok { + t.Fatalf("store is %T, want TurnLedger", store) + } + ctx := t.Context() + const ( + firstSession = "a\x00b" + firstSource = "c" + secondSession = "a" + secondSource = "b\x00c" + ) + collidingLegacyID := legacyCanonicalTurnID(firstSession, firstSource) + if _, claim, err := ledger.ClaimTurn(ctx, TurnRecord{ + TurnID: collidingLegacyID, SessionID: firstSession, SourceID: firstSource, + Status: TurnStatusAccepted, OwnerID: "first-process", + }); err != nil || claim != TurnClaimOwned { + t.Fatalf("seed colliding turn = %q, %v", claim, err) + } + + runner := NewTurnRunner(TurnRunnerConfig{Ledger: ledger}) + got, err := runner.canonicalTurnID(ctx, secondSession, secondSource) + if err != nil { + t.Fatalf("canonicalTurnID: %v", err) + } + want := CanonicalTurnID(secondSession, secondSource) + if got != want || got == collidingLegacyID { + t.Fatalf("canonicalTurnID = %q, want isolated ID %q (legacy collision %q)", + got, want, collidingLegacyID) + } +} diff --git a/internal/gateway/turns.go b/internal/gateway/turns.go index f6f92e233..506ab55ad 100644 --- a/internal/gateway/turns.go +++ b/internal/gateway/turns.go @@ -76,16 +76,14 @@ func NewTurns(log *slog.Logger) *Turns { // // Submit never blocks and never rejects: messages queue. func (t *Turns) Submit(ctx context.Context, session string, run func(context.Context)) { + turnCtx, cancel := context.WithCancel(ctx) + t.mu.Lock() l, ok := t.lanes[session] if !ok { l = &lane{} t.lanes[session] = l } - t.mu.Unlock() - - turnCtx, cancel := context.WithCancel(ctx) - l.mu.Lock() l.pending = append(l.pending, queued{run: func() { run(turnCtx) }, cancel: cancel}) depth := len(l.pending) @@ -94,9 +92,10 @@ func (t *Turns) Submit(ctx context.Context, session string, run func(context.Con l.working = true } l.mu.Unlock() + t.mu.Unlock() if start { - go l.serve(t.log.With("session", session)) + go t.serve(session, l) } if depth > 1 { t.log.Info("chat turn queued", "session", session, "queue_depth", depth) @@ -154,7 +153,8 @@ func (t *Turns) Queued(session string) int { return len(l.pending) } -// lane returns the session's lane, or nil if it has never been used. +// lane returns the session's active lane, or nil if it has never been used or +// its completed lane has already retired. func (t *Turns) lane(session string) *lane { t.mu.Lock() defer t.mu.Unlock() @@ -164,19 +164,45 @@ func (t *Turns) lane(session string) *lane { // serve drains the lane's backlog one turn at a time and exits once it is // empty. Submit starts a fresh worker when more arrives, so an idle session // costs no goroutine. -func (l *lane) serve(log *slog.Logger) { +func (t *Turns) serve(session string, l *lane) { + log := t.log.With("session", session) for { next, ok := l.begin() if !ok { - return + if t.retire(session, l) { + return + } + continue } l.runOne(next, log) } } +// retire removes an idle lane while holding the same two locks Submit uses to +// find and append to it. Without that joint critical section, Submit could find +// the old lane immediately before cleanup deleted it and strand the new turn on +// a worker that had already exited. +func (t *Turns) retire(session string, l *lane) bool { + t.mu.Lock() + defer t.mu.Unlock() + + current, ok := t.lanes[session] + if !ok || current != l { + return true + } + l.mu.Lock() + defer l.mu.Unlock() + if len(l.pending) != 0 { + return false + } + l.working = false + delete(t.lanes, session) + return true +} + // begin pops the oldest queued turn and publishes its cancel in the same -// critical section, reporting false when the backlog is empty -- at which -// point it also retires the worker. +// critical section. It reports false when the backlog is empty; the owning +// Turns then retires the worker and map entry under both locks. // // The pop and the publish must happen together. Were the turn popped first // and its cancel published after, a Stop landing in between would see an @@ -187,11 +213,8 @@ func (l *lane) begin() (queued, bool) { defer l.mu.Unlock() if len(l.pending) == 0 { - // Retire under the same lock Submit uses to decide whether a - // worker is live, so a turn arriving now either starts a new - // worker or is picked up before this one retires -- never - // neither, which would strand the backlog forever. - l.working = false + // The owning Turns retires the lane under both the map and lane + // locks. Changing working here would race a concurrent Submit. return queued{}, false } diff --git a/internal/gateway/turns_internal_test.go b/internal/gateway/turns_internal_test.go new file mode 100644 index 000000000..4f3c73c7a --- /dev/null +++ b/internal/gateway/turns_internal_test.go @@ -0,0 +1,52 @@ +package gateway + +import ( + "context" + "log/slog" + "testing" + "time" +) + +func TestTurnsRetiresIdleSessionLanes(t *testing.T) { + t.Parallel() + + turns := NewTurns(slog.New(slog.DiscardHandler)) + const sessions = 100 + done := make(chan struct{}, sessions) + for i := range sessions { + turns.Submit(context.Background(), string(rune(i)), func(context.Context) { + done <- struct{}{} + }) + } + for range sessions { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("submitted turn did not finish") + } + } + + deadline := time.Now().Add(2 * time.Second) + for { + turns.mu.Lock() + remaining := len(turns.lanes) + turns.mu.Unlock() + if remaining == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%d idle session lanes remain allocated", remaining) + } + time.Sleep(time.Millisecond) + } + + // A retired session must remain reusable; cleanup cannot strand a turn + // between looking up the old lane and creating its replacement. + reused := make(chan struct{}) + turns.Submit(context.Background(), string(rune(0)), func(context.Context) { close(reused) }) + select { + case <-reused: + case <-time.After(2 * time.Second): + t.Fatal("retired session did not accept a new turn") + } +}