fix(gateway): claim the session slot before writing it, not after
resolve saved a newly created session to the store and only then asked the cache who had won a concurrent race. The loser had already persisted a session that nothing pointed at: unreachable, but still listed by /topic and /sessions as "(untitled)", and -- now that resolution goes by recency -- resurrectable on a later restart. Confirmed by forcing the interleaving: the store held 2 sessions where it should hold 1. The claim now happens under the same double-check, ahead of the store write, so only the winner writes. A failed Save releases the claim, or the cache would point every later message in that chat at a session that was never stored. Refs archie-core-x35
This commit is contained in:
1 parent
a3b9eded5a
commit
44d43ef286
2 files changed
+166
-1
No files matched your search
@@ -78,7 +78,18 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
||||
|
||||
// Create a new session with a deterministic key so it matches the
|
||||
// legacy sessionKey format used by the LLM responder.
|
||||
//
|
||||
// Claim the cache slot before writing to the store, not after. Saving
|
||||
// first and asking the cache who won afterwards meant the loser of a race
|
||||
// had already persisted a session that nothing pointed at -- unreachable,
|
||||
// but still listed by /topic and /sessions as "(untitled)", and
|
||||
// resurrectable by a later restart now that resolution goes by recency.
|
||||
// Whoever holds the slot is the only one that writes.
|
||||
id = sessionKeyFromFields(channelID, threadID)
|
||||
if winner, won := t.claimActive(key, id); !won {
|
||||
return winner, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
sc := SessionContext{
|
||||
SessionID: id,
|
||||
@@ -92,9 +103,38 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
||||
LastActiveAt: now,
|
||||
}
|
||||
if err := t.sessions.Save(ctx, sc); err != nil {
|
||||
// Drop the claim: leaving it would point every later message in this
|
||||
// chat at a session that was never stored.
|
||||
t.releaseActive(key, id)
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
return t.cacheActive(key, id), nil
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// claimActive reserves key for sessionID unless another caller already holds
|
||||
// it. It returns the winning session ID and whether this caller won.
|
||||
//
|
||||
// This is the same double-check cacheActive performs, moved ahead of the
|
||||
// store write so only the winner writes.
|
||||
func (t *sessionTracker) claimActive(key, sessionID string) (string, bool) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if existing, ok := t.active[key]; ok && existing != "" {
|
||||
return existing, false
|
||||
}
|
||||
t.active[key] = sessionID
|
||||
return sessionID, true
|
||||
}
|
||||
|
||||
// releaseActive drops a claim that could not be completed. It only removes
|
||||
// the entry when it is still this caller's, so a racing setActive that has
|
||||
// since replaced it is left alone.
|
||||
func (t *sessionTracker) releaseActive(key, sessionID string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.active[key] == sessionID {
|
||||
delete(t.active, key)
|
||||
}
|
||||
}
|
||||
|
||||
// cacheActive records the resolved session for a channel+thread. A racing
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// hookStore fires a callback immediately after GetByChannel returns, which is
|
||||
// the point inside resolve where a racing command can land.
|
||||
type hookStore struct {
|
||||
SessionStore
|
||||
afterGetByChannel func()
|
||||
}
|
||||
|
||||
func (h *hookStore) GetByChannel(ctx context.Context, platform, channelID string) ([]SessionContext, error) {
|
||||
out, err := h.SessionStore.GetByChannel(ctx, platform, channelID)
|
||||
if h.afterGetByChannel != nil {
|
||||
fn := h.afterGetByChannel
|
||||
h.afterGetByChannel = nil // fire once; the callback itself resolves
|
||||
fn()
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
|
||||
// resolve saved its new session to the store before asking the cache who won,
|
||||
// so the loser of that race left a session behind that nothing points at. It
|
||||
// still appeared in /topic and /sessions listings as "(untitled)", and with
|
||||
// recency-based resolution it could be resurrected on a later restart.
|
||||
//
|
||||
// The daemon really does run commands on lane goroutines while submitTurn
|
||||
// resolves on the channel worker, so this interleaving is reachable.
|
||||
func TestResolveDoesNotOrphanASessionItLoses(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
inner := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = inner.Close() })
|
||||
store := &hookStore{SessionStore: inner}
|
||||
|
||||
r := NewRouter(nil, nil, "telegram")
|
||||
r.InitSessions(store)
|
||||
r.Identity = "archie"
|
||||
|
||||
// While resolve is between its store lookup and its own write, /new
|
||||
// creates a session and makes it active.
|
||||
var newSessionID string
|
||||
store.afterGetByChannel = func() {
|
||||
if _, err := r.Route(ctx, Message{Text: "/new", ChannelID: "chat-y"}); err != nil {
|
||||
t.Errorf("Route(/new): %v", err)
|
||||
return
|
||||
}
|
||||
newSessionID = r.sessionTracker.getActive("chat-y", "")
|
||||
}
|
||||
|
||||
got, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-y", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve: %v", err)
|
||||
}
|
||||
|
||||
if newSessionID == "" {
|
||||
t.Fatal("the racing /new did not produce a session; the interleaving did not happen")
|
||||
}
|
||||
if got != newSessionID {
|
||||
t.Errorf("resolve = %q, want the racing winner %q", got, newSessionID)
|
||||
}
|
||||
|
||||
sessions, err := inner.List(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("List: %v", err)
|
||||
}
|
||||
if len(sessions) != 1 {
|
||||
t.Fatalf("store holds %d sessions (%v), want only the one that won: "+
|
||||
"the loser was persisted and is now unreachable",
|
||||
len(sessions), sessionIDs(sessions))
|
||||
}
|
||||
if sessions[0].SessionID != newSessionID {
|
||||
t.Errorf("stored session = %q, want %q", sessions[0].SessionID, newSessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed Save must not leave the cache pointing at a session that does not
|
||||
// exist, or every later message in that chat addresses a phantom.
|
||||
func TestResolveReleasesItsClaimWhenSaveFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
inner := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = inner.Close() })
|
||||
store := &failingSaveStore{SessionStore: inner, fail: true}
|
||||
|
||||
r := NewRouter(nil, nil, "telegram")
|
||||
r.InitSessions(store)
|
||||
|
||||
if _, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-z", ""); err == nil {
|
||||
t.Fatal("resolve succeeded despite a failing Save")
|
||||
}
|
||||
if got := r.sessionTracker.getActive("chat-z", ""); got != "" {
|
||||
t.Fatalf("getActive = %q after a failed create, want empty: the cache "+
|
||||
"points at a session that was never stored", got)
|
||||
}
|
||||
|
||||
// With the store working again, the next attempt must succeed.
|
||||
store.fail = false
|
||||
id, err := r.sessionTracker.resolve(ctx, "telegram", "archie", "chat-z", "")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve after recovery: %v", err)
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("resolve returned an empty session ID after recovery")
|
||||
}
|
||||
}
|
||||
|
||||
type failingSaveStore struct {
|
||||
SessionStore
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (f *failingSaveStore) Save(ctx context.Context, sc SessionContext) error {
|
||||
if f.fail {
|
||||
return errSaveRefused
|
||||
}
|
||||
return f.SessionStore.Save(ctx, sc)
|
||||
}
|
||||
|
||||
var errSaveRefused = errTest("save refused")
|
||||
|
||||
type errTest string
|
||||
|
||||
func (e errTest) Error() string { return string(e) }
|
||||
Reference in new issue
Block a user