fix(gateway): claim the session slot before writing it, not after
resolve saved a newly created session to the store and only then asked the cache who had won a concurrent race. The loser had already persisted a session that nothing pointed at: unreachable, but still listed by /topic and /sessions as "(untitled)", and -- now that resolution goes by recency -- resurrectable on a later restart. Confirmed by forcing the interleaving: the store held 2 sessions where it should hold 1. The claim now happens under the same double-check, ahead of the store write, so only the winner writes. A failed Save releases the claim, or the cache would point every later message in that chat at a session that was never stored. Refs archie-core-x35
This commit is contained in:
1 parent
a3b9eded5a
commit
44d43ef286
2 files changed
+166
-1
No files matched your search
@@ -78,7 +78,18 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
|||||||
|
|
||||||
// Create a new session with a deterministic key so it matches the
|
// Create a new session with a deterministic key so it matches the
|
||||||
// legacy sessionKey format used by the LLM responder.
|
// legacy sessionKey format used by the LLM responder.
|
||||||
|
//
|
||||||
|
// Claim the cache slot before writing to the store, not after. Saving
|
||||||
|
// first and asking the cache who won afterwards meant the loser of a race
|
||||||
|
// had already persisted a session that nothing pointed at -- unreachable,
|
||||||
|
// but still listed by /topic and /sessions as "(untitled)", and
|
||||||
|
// resurrectable by a later restart now that resolution goes by recency.
|
||||||
|
// Whoever holds the slot is the only one that writes.
|
||||||
id = sessionKeyFromFields(channelID, threadID)
|
id = sessionKeyFromFields(channelID, threadID)
|
||||||
|
if winner, won := t.claimActive(key, id); !won {
|
||||||
|
return winner, nil
|
||||||
|
}
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
sc := SessionContext{
|
sc := SessionContext{
|
||||||
SessionID: id,
|
SessionID: id,
|
||||||
@@ -92,9 +103,38 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
|||||||
LastActiveAt: now,
|
LastActiveAt: now,
|
||||||
}
|
}
|
||||||
if err := t.sessions.Save(ctx, sc); err != nil {
|
if err := t.sessions.Save(ctx, sc); err != nil {
|
||||||
|
// Drop the claim: leaving it would point every later message in this
|
||||||
|
// chat at a session that was never stored.
|
||||||
|
t.releaseActive(key, id)
|
||||||
return "", fmt.Errorf("create session: %w", err)
|
return "", fmt.Errorf("create session: %w", err)
|
||||||
}
|
}
|
||||||
return t.cacheActive(key, id), nil
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// claimActive reserves key for sessionID unless another caller already holds
|
||||||
|
// it. It returns the winning session ID and whether this caller won.
|
||||||
|
//
|
||||||
|
// This is the same double-check cacheActive performs, moved ahead of the
|
||||||
|
// store write so only the winner writes.
|
||||||
|
func (t *sessionTracker) claimActive(key, sessionID string) (string, bool) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if existing, ok := t.active[key]; ok && existing != "" {
|
||||||
|
return existing, false
|
||||||
|
}
|
||||||
|
t.active[key] = sessionID
|
||||||
|
return sessionID, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// releaseActive drops a claim that could not be completed. It only removes
|
||||||
|
// the entry when it is still this caller's, so a racing setActive that has
|
||||||
|
// since replaced it is left alone.
|
||||||
|
func (t *sessionTracker) releaseActive(key, sessionID string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if t.active[key] == sessionID {
|
||||||
|
delete(t.active, key)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// cacheActive records the resolved session for a channel+thread. A racing
|
// cacheActive records the resolved session for a channel+thread. A racing
|
||||||
|
|||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package gateway
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// hookStore fires a callback immediately after GetByChannel returns, which is
|
||||||
|
// the point inside resolve where a racing command can land.
|
||||||
|
type hookStore struct {
|
||||||
|
SessionStore
|
||||||
|
afterGetByChannel func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *hookStore) GetByChannel(ctx context.Context, platform, channelID string) ([]SessionContext, error) {
|
||||||
|
out, err := h.SessionStore.GetByChannel(ctx, platform, channelID)
|
||||||
|
if h.afterGetByChannel != nil {
|
||||||
|
fn := h.afterGetByChannel
|
||||||
|
h.afterGetByChannel = nil // fire once; the callback itself resolves
|
||||||
|
fn()
|
||||||
|
}
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve saved its new session to the store before asking the cache who won,
|
||||||
|
// so the loser of that race left a session behind that nothing points at. It
|
||||||
|
// still appeared in /topic and /sessions listings as "(untitled)", and with
|
||||||
|
// recency-based resolution it could be resurrected on a later restart.
|
||||||
|
//
|
||||||
|
// The daemon really does run commands on lane goroutines while submitTurn
|
||||||
|
// resolves on the channel worker, so this interleaving is reachable.
|
||||||
|
func TestResolveDoesNotOrphanASessionItLoses(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
inner := NewSessionStoreMemory("test-node")
|
||||||
|
t.Cleanup(func() { _ = inner.Close() })
|
||||||
|
store := &hookStore{SessionStore: inner}
|
||||||
|
|
||||||
|
r := NewRouter(nil, nil, "telegram")
|
||||||
|
r.InitSessions(store)
|
||||||
|
r.Identity = "archie"
|
||||||
|
|
||||||
|
// While resolve is between its store lookup and its own write, /new
|
||||||
|
// creates a session and makes it active.
|
||||||
|
var newSessionID string
|
||||||
|
store.afterGetByChannel = func() {
|
||||||
|
if _, err := r.Route(ctx, Message{Text: "/new", ChannelID: "chat-y"}); err != nil {
|
||||||
|
t.Errorf("Route(/new): %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
newSessionID = r.sessionTracker.getActive("chat-y", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-y", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if newSessionID == "" {
|
||||||
|
t.Fatal("the racing /new did not produce a session; the interleaving did not happen")
|
||||||
|
}
|
||||||
|
if got != newSessionID {
|
||||||
|
t.Errorf("resolve = %q, want the racing winner %q", got, newSessionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessions, err := inner.List(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List: %v", err)
|
||||||
|
}
|
||||||
|
if len(sessions) != 1 {
|
||||||
|
t.Fatalf("store holds %d sessions (%v), want only the one that won: "+
|
||||||
|
"the loser was persisted and is now unreachable",
|
||||||
|
len(sessions), sessionIDs(sessions))
|
||||||
|
}
|
||||||
|
if sessions[0].SessionID != newSessionID {
|
||||||
|
t.Errorf("stored session = %q, want %q", sessions[0].SessionID, newSessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed Save must not leave the cache pointing at a session that does not
|
||||||
|
// exist, or every later message in that chat addresses a phantom.
|
||||||
|
func TestResolveReleasesItsClaimWhenSaveFails(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
inner := NewSessionStoreMemory("test-node")
|
||||||
|
t.Cleanup(func() { _ = inner.Close() })
|
||||||
|
store := &failingSaveStore{SessionStore: inner, fail: true}
|
||||||
|
|
||||||
|
r := NewRouter(nil, nil, "telegram")
|
||||||
|
r.InitSessions(store)
|
||||||
|
|
||||||
|
if _, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-z", ""); err == nil {
|
||||||
|
t.Fatal("resolve succeeded despite a failing Save")
|
||||||
|
}
|
||||||
|
if got := r.sessionTracker.getActive("chat-z", ""); got != "" {
|
||||||
|
t.Fatalf("getActive = %q after a failed create, want empty: the cache "+
|
||||||
|
"points at a session that was never stored", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// With the store working again, the next attempt must succeed.
|
||||||
|
store.fail = false
|
||||||
|
id, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-z", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve after recovery: %v", err)
|
||||||
|
}
|
||||||
|
if id == "" {
|
||||||
|
t.Fatal("resolve returned an empty session ID after recovery")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type failingSaveStore struct {
|
||||||
|
SessionStore
|
||||||
|
fail bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *failingSaveStore) Save(ctx context.Context, sc SessionContext) error {
|
||||||
|
if f.fail {
|
||||||
|
return errSaveRefused
|
||||||
|
}
|
||||||
|
return f.SessionStore.Save(ctx, sc)
|
||||||
|
}
|
||||||
|
|
||||||
|
var errSaveRefused = errTest("save refused")
|
||||||
|
|
||||||
|
type errTest string
|
||||||
|
|
||||||
|
func (e errTest) Error() string { return string(e) }
|
||||||
Reference in new issue
Block a user