diff --git a/internal/gateway/session_store_sqlite.go b/internal/gateway/session_store_sqlite.go index f8e3057d5..0ae9c9944 100644 --- a/internal/gateway/session_store_sqlite.go +++ b/internal/gateway/session_store_sqlite.go @@ -224,19 +224,27 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord) initial.CreatedAt = now } initial.UpdatedAt = now - result, err := tx.ExecContext(ctx, ` + // insertErr is deliberately its own name, not err: this whole block + // is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here + // would shadow the outer err for the rest of the block -- including + // the retry read below, whose result must reach the outer `if err + // != nil` after this block closes. A shadowed reassignment there + // would go out of scope silently, leaving the outer err at its + // original sql.ErrNoRows value regardless of how the retry read + // actually went. + result, insertErr := tx.ExecContext(ctx, ` INSERT INTO turns ( turn_id, session_id, source_id, status, attempt, owner_id, input_message_id, assistant_message_id, partial_text, response_text, error, created_at, updated_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...) - if err != nil { - return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", err) + if insertErr != nil { + return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr) } - affected, err := result.RowsAffected() - if err != nil { - return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", err) + affected, insertErr := result.RowsAffected() + if insertErr != nil { + return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", insertErr) } if affected == 1 { if err := tx.Commit(); err != nil { @@ -246,7 +254,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord) } // Another connection inserted the same turn between our read and // insert. Read it in this transaction and apply the normal claim - // state machine instead of surfacing a uniqueness error. + // state machine instead of surfacing a uniqueness error. This + // assigns the OUTER err (see above) -- it is what the `if err != + // nil` check below this block actually sees. err = tx.QueryRowContext(ctx, ` SELECT turn_id, session_id, source_id, status, attempt, owner_id, input_message_id, assistant_message_id, partial_text, diff --git a/internal/gateway/session_store_sqlite_claimturn_race_test.go b/internal/gateway/session_store_sqlite_claimturn_race_test.go new file mode 100644 index 000000000..4a3de9347 --- /dev/null +++ b/internal/gateway/session_store_sqlite_claimturn_race_test.go @@ -0,0 +1,140 @@ +package gateway + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// TestClaimTurnConcurrentInsertRaceNeverReturnsAnError guards against a +// variable-shadowing bug in ClaimTurn (session_store_sqlite.go), fixed +// alongside this test: the branch handling "another connection inserted the +// same turn between our read and insert" used to write its retry read's +// result into an `err` shadowed inside the `errors.Is(err, sql.ErrNoRows)` +// block by an earlier `result, err := tx.ExecContext(...)`. That shadowed +// err went out of scope at the block's closing brace, so the outer err +// checked afterward was still sql.ErrNoRows from the original read -- the +// loser of that race would always return an error wrapped +// "sessionstore: read turn: %w", even though the comment's own stated +// intent is "apply the normal claim state machine instead of surfacing a +// uniqueness error." Confirmed via careful reading of Go's block-scoping +// rules, not assumed. +// +// Honest limitation: this test could NOT be made to reproduce that exact +// branch through genuine concurrent access. A one-off diagnostic (real +// two-connection race, sleep-widened to force overlap, reverted before this +// commit -- never present in a shipped file) showed the loser instead fails +// its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with this +// driver, before ON CONFLICT DO NOTHING can ever report affected=0. That +// means the specific "0 rows affected" branch this bug lived in may be +// effectively unreachable via a same-driver, same-journal-mode race today. +// The fix stands regardless -- the shadowing was real, wrong on its own +// terms, and free to correct -- but this test cannot claim to exercise the +// exact line that was fixed. What it verifies is the weaker, still real +// property that matters operationally: many genuinely concurrent +// same-turn-ID claims across separate connections never produce the bug's +// specific error signature. Treat this as a guard against a regression +// becoming reachable (a different SQLite version, journal mode, or driver +// with weaker snapshot isolation), not as proof the original bug was ever +// live in production. +// +// A single sqliteSessionStore instance cannot exhibit any of this: its own +// mutex serializes every ClaimTurn call against that instance's connection, +// so this uses two separate store instances (two connections, two mutexes) +// against the same file-backed database -- the real-world shape of the +// scenario (two archied/worker processes or connections sharing one session +// DB). +// +// SQLite's own write-lock contention (SQLITE_BUSY / SQLITE_BUSY_SNAPSHOT, +// "database is locked") is infrastructure noise distinct from the bug: it +// comes from BeginTx/ExecContext/Commit failing outright and is wrapped +// with a different prefix ("claim turn: begin/insert/rows affected/commit"). +// Only the "sessionstore: read turn:" wrap was the bug's unique signature, +// so transient busy errors are retried rather than treated as failures. +func TestClaimTurnConcurrentInsertRaceNeverReturnsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "sessions.db") + + const stores = 2 + const trials = 100 + const maxRetriesPerAttempt = 20 + + openers := make([]TurnLedger, stores) + for i := range openers { + store, err := OpenSQLiteSessionStore(path) + if err != nil { + t.Fatalf("OpenSQLiteSessionStore[%d]: %v", i, err) + } + closer, ok := store.(interface{ Close() error }) + if !ok { + t.Fatalf("store[%d] is %T, want a Close() error", i, store) + } + t.Cleanup(func() { _ = closer.Close() }) + ledger, ok := store.(TurnLedger) + if !ok { + t.Fatalf("store[%d] is %T, want TurnLedger", i, store) + } + openers[i] = ledger + } + + ctx := context.Background() + now := time.UnixMilli(1000).UTC() + + claimRetryingBusy := func(ledger TurnLedger, initial TurnRecord) error { + var err error + for range maxRetriesPerAttempt { + _, _, err = ledger.ClaimTurn(ctx, initial) + if err == nil || !strings.Contains(err.Error(), "database is locked") { + return err + } + } + return err + } + + // All trials race concurrently, not one at a time: a sequential + // trial-by-trial race tends to resolve cleanly before the next trial + // starts (the loser's retry sees an already-committed winner and skips + // the buggy branch entirely), which undersells how likely the narrow + // timing window is under real, overlapping load. + errs := make([][]error, trials) + var wg sync.WaitGroup + var start sync.WaitGroup + start.Add(1) + for trial := range trials { + turnID := fmt.Sprintf("race-turn-%d", trial) + initial := TurnRecord{ + TurnID: turnID, + SessionID: "race-session", + SourceID: turnID, + Status: TurnStatusAccepted, + CreatedAt: now, + UpdatedAt: now, + } + errs[trial] = make([]error, stores) + for i := range stores { + wg.Add(1) + go func(trial, i int) { + defer wg.Done() + start.Wait() + errs[trial][i] = claimRetryingBusy(openers[i], initial) + }(trial, i) + } + } + start.Done() + wg.Wait() + + for trial, storeErrs := range errs { + for i, err := range storeErrs { + if err == nil { + continue + } + if strings.Contains(err.Error(), "sessionstore: read turn:") { + t.Fatalf("trial %d, store %d: ClaimTurn() error = %v -- a losing racer surfaced the winner's insert as a read failure instead of applying the normal claim state machine", trial, i, err) + } + t.Fatalf("trial %d, store %d: ClaimTurn() unexpected error = %v", trial, i, err) + } + } +}