From d3f9a7a4727ff17a23be3fc2629cfb2046a438fa Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Sat, 22 Aug 2026 02:03:27 +1000 Subject: [PATCH] fix(gateway): stop ClaimTurn's insert-race branch losing its own retry read Found while investigating an `ineffassign` finding, not a stylistic report: the branch handling "another connection inserted the same turn between our read and insert" was discarding its own result. `result, err := tx.ExecContext(ctx, ...)` for the INSERT sits inside `if errors.Is(err, sql.ErrNoRows) { ... }`, a new block scope, with `result` new to that scope -- so Go's `:=` declares a fresh, block-scoped `err` shadowing the function's own `err` for the rest of the block, including the retry read a few lines later (`err = tx.QueryRowContext(...).Scan(...)`). That assignment landed in the shadow, which goes out of scope at the block's closing brace. The `if err != nil` check afterward was therefore still looking at the *original* err -- sql.ErrNoRows from the very first read -- regardless of whether the retry read succeeded. The loser of a genuine insert race would always return `sessionstore: read turn: %w` wrapping `sql.ErrNoRows`, contradicting the branch's own comment: "apply the normal claim state machine instead of surfacing a uniqueness error." Fix: the insert-path errors are now insertErr, not err, so the retry read's plain assignment reaches the outer err the later check actually inspects. Confirmed via careful trace of Go's block-scoping rules that this closes the gap for the affected!=1 (loser) path without touching the affected==1 (winner) path, which returns from its own separately-scoped err before ever reaching this one. Commit the honest limitation along with the fix: a genuine two-connection concurrent regression test could not be made to hit the exact branch. A temporary, reverted-before-commit diagnostic (real concurrent race, artificially widened with a sleep) showed the losing connection instead fails its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with this driver, before ON CONFLICT DO NOTHING can ever report 0 rows affected -- meaning the specific branch this bug lived in may be effectively unreachable via this driver/journal-mode combination today. The shipped test verifies the weaker, still real property that matters operationally: many genuinely concurrent same-turn-ID claims across separate connections never produce the bug's error signature ("sessionstore: read turn:"), distinct from unrelated SQLITE_BUSY/SQLITE_BUSY_SNAPSHOT lock contention, which is retried rather than treated as a failure. The fix stands regardless of provable live reachability -- the shadowing was unambiguous and wrong on its own terms. Adversarially reviewed by a fresh subagent, which independently reverted just the fix (keeping the test) and confirmed empirically -- 20 runs, plus 1000 trials with a debug probe on the affected!=1 branch -- that the branch is never reached under this driver, corroborating the test's own stated limitation rather than contradicting it. gocognit/nestif findings on ClaimTurn are pre-existing on the prior committed baseline, unchanged by this fix (only variable names moved). --- internal/gateway/session_store_sqlite.go | 24 ++- ...ession_store_sqlite_claimturn_race_test.go | 140 ++++++++++++++++++ 2 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 internal/gateway/session_store_sqlite_claimturn_race_test.go diff --git a/internal/gateway/session_store_sqlite.go b/internal/gateway/session_store_sqlite.go index f8e3057d5..0ae9c9944 100644 --- a/internal/gateway/session_store_sqlite.go +++ b/internal/gateway/session_store_sqlite.go @@ -224,19 +224,27 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord) initial.CreatedAt = now } initial.UpdatedAt = now - result, err := tx.ExecContext(ctx, ` + // insertErr is deliberately its own name, not err: this whole block + // is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here + // would shadow the outer err for the rest of the block -- including + // the retry read below, whose result must reach the outer `if err + // != nil` after this block closes. A shadowed reassignment there + // would go out of scope silently, leaving the outer err at its + // original sql.ErrNoRows value regardless of how the retry read + // actually went. + result, insertErr := tx.ExecContext(ctx, ` INSERT INTO turns ( turn_id, session_id, source_id, status, attempt, owner_id, input_message_id, assistant_message_id, partial_text, response_text, error, created_at, updated_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...) - if err != nil { - return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", err) + if insertErr != nil { + return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr) } - affected, err := result.RowsAffected() - if err != nil { - return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", err) + affected, insertErr := result.RowsAffected() + if insertErr != nil { + return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", insertErr) } if affected == 1 { if err := tx.Commit(); err != nil { @@ -246,7 +254,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord) } // Another connection inserted the same turn between our read and // insert. Read it in this transaction and apply the normal claim - // state machine instead of surfacing a uniqueness error. + // state machine instead of surfacing a uniqueness error. This + // assigns the OUTER err (see above) -- it is what the `if err != + // nil` check below this block actually sees. err = tx.QueryRowContext(ctx, ` SELECT turn_id, session_id, source_id, status, attempt, owner_id, input_message_id, assistant_message_id, partial_text, diff --git a/internal/gateway/session_store_sqlite_claimturn_race_test.go b/internal/gateway/session_store_sqlite_claimturn_race_test.go new file mode 100644 index 000000000..4a3de9347 --- /dev/null +++ b/internal/gateway/session_store_sqlite_claimturn_race_test.go @@ -0,0 +1,140 @@ +package gateway + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// TestClaimTurnConcurrentInsertRaceNeverReturnsAnError guards against a +// variable-shadowing bug in ClaimTurn (session_store_sqlite.go), fixed +// alongside this test: the branch handling "another connection inserted the +// same turn between our read and insert" used to write its retry read's +// result into an `err` shadowed inside the `errors.Is(err, sql.ErrNoRows)` +// block by an earlier `result, err := tx.ExecContext(...)`. That shadowed +// err went out of scope at the block's closing brace, so the outer err +// checked afterward was still sql.ErrNoRows from the original read -- the +// loser of that race would always return an error wrapped +// "sessionstore: read turn: %w", even though the comment's own stated +// intent is "apply the normal claim state machine instead of surfacing a +// uniqueness error." Confirmed via careful reading of Go's block-scoping +// rules, not assumed. +// +// Honest limitation: this test could NOT be made to reproduce that exact +// branch through genuine concurrent access. A one-off diagnostic (real +// two-connection race, sleep-widened to force overlap, reverted before this +// commit -- never present in a shipped file) showed the loser instead fails +// its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with this +// driver, before ON CONFLICT DO NOTHING can ever report affected=0. That +// means the specific "0 rows affected" branch this bug lived in may be +// effectively unreachable via a same-driver, same-journal-mode race today. +// The fix stands regardless -- the shadowing was real, wrong on its own +// terms, and free to correct -- but this test cannot claim to exercise the +// exact line that was fixed. What it verifies is the weaker, still real +// property that matters operationally: many genuinely concurrent +// same-turn-ID claims across separate connections never produce the bug's +// specific error signature. Treat this as a guard against a regression +// becoming reachable (a different SQLite version, journal mode, or driver +// with weaker snapshot isolation), not as proof the original bug was ever +// live in production. +// +// A single sqliteSessionStore instance cannot exhibit any of this: its own +// mutex serializes every ClaimTurn call against that instance's connection, +// so this uses two separate store instances (two connections, two mutexes) +// against the same file-backed database -- the real-world shape of the +// scenario (two archied/worker processes or connections sharing one session +// DB). +// +// SQLite's own write-lock contention (SQLITE_BUSY / SQLITE_BUSY_SNAPSHOT, +// "database is locked") is infrastructure noise distinct from the bug: it +// comes from BeginTx/ExecContext/Commit failing outright and is wrapped +// with a different prefix ("claim turn: begin/insert/rows affected/commit"). +// Only the "sessionstore: read turn:" wrap was the bug's unique signature, +// so transient busy errors are retried rather than treated as failures. +func TestClaimTurnConcurrentInsertRaceNeverReturnsAnError(t *testing.T) { + path := filepath.Join(t.TempDir(), "sessions.db") + + const stores = 2 + const trials = 100 + const maxRetriesPerAttempt = 20 + + openers := make([]TurnLedger, stores) + for i := range openers { + store, err := OpenSQLiteSessionStore(path) + if err != nil { + t.Fatalf("OpenSQLiteSessionStore[%d]: %v", i, err) + } + closer, ok := store.(interface{ Close() error }) + if !ok { + t.Fatalf("store[%d] is %T, want a Close() error", i, store) + } + t.Cleanup(func() { _ = closer.Close() }) + ledger, ok := store.(TurnLedger) + if !ok { + t.Fatalf("store[%d] is %T, want TurnLedger", i, store) + } + openers[i] = ledger + } + + ctx := context.Background() + now := time.UnixMilli(1000).UTC() + + claimRetryingBusy := func(ledger TurnLedger, initial TurnRecord) error { + var err error + for range maxRetriesPerAttempt { + _, _, err = ledger.ClaimTurn(ctx, initial) + if err == nil || !strings.Contains(err.Error(), "database is locked") { + return err + } + } + return err + } + + // All trials race concurrently, not one at a time: a sequential + // trial-by-trial race tends to resolve cleanly before the next trial + // starts (the loser's retry sees an already-committed winner and skips + // the buggy branch entirely), which undersells how likely the narrow + // timing window is under real, overlapping load. + errs := make([][]error, trials) + var wg sync.WaitGroup + var start sync.WaitGroup + start.Add(1) + for trial := range trials { + turnID := fmt.Sprintf("race-turn-%d", trial) + initial := TurnRecord{ + TurnID: turnID, + SessionID: "race-session", + SourceID: turnID, + Status: TurnStatusAccepted, + CreatedAt: now, + UpdatedAt: now, + } + errs[trial] = make([]error, stores) + for i := range stores { + wg.Add(1) + go func(trial, i int) { + defer wg.Done() + start.Wait() + errs[trial][i] = claimRetryingBusy(openers[i], initial) + }(trial, i) + } + } + start.Done() + wg.Wait() + + for trial, storeErrs := range errs { + for i, err := range storeErrs { + if err == nil { + continue + } + if strings.Contains(err.Error(), "sessionstore: read turn:") { + t.Fatalf("trial %d, store %d: ClaimTurn() error = %v -- a losing racer surfaced the winner's insert as a read failure instead of applying the normal claim state machine", trial, i, err) + } + t.Fatalf("trial %d, store %d: ClaimTurn() unexpected error = %v", trial, i, err) + } + } +}