fix(gateway): stop ClaimTurn's insert-race branch losing its own retry read

Found while investigating an `ineffassign` finding, not a stylistic report:
the branch handling "another connection inserted the same turn between our
read and insert" was discarding its own result.

`result, err := tx.ExecContext(ctx, ...)` for the INSERT sits inside
`if errors.Is(err, sql.ErrNoRows) { ... }`, a new block scope, with `result`
new to that scope -- so Go's `:=` declares a fresh, block-scoped `err`
shadowing the function's own `err` for the rest of the block, including the
retry read a few lines later (`err = tx.QueryRowContext(...).Scan(...)`).
That assignment landed in the shadow, which goes out of scope at the block's
closing brace. The `if err != nil` check afterward was therefore still
looking at the *original* err -- sql.ErrNoRows from the very first read --
regardless of whether the retry read succeeded. The loser of a genuine
insert race would always return `sessionstore: read turn: %w` wrapping
`sql.ErrNoRows`, contradicting the branch's own comment: "apply the normal
claim state machine instead of surfacing a uniqueness error."

Fix: the insert-path errors are now insertErr, not err, so the retry read's
plain assignment reaches the outer err the later check actually inspects.
Confirmed via careful trace of Go's block-scoping rules that this closes the
gap for the affected!=1 (loser) path without touching the affected==1
(winner) path, which returns from its own separately-scoped err before ever
reaching this one.

Commit the honest limitation along with the fix: a genuine two-connection
concurrent regression test could not be made to hit the exact branch. A
temporary, reverted-before-commit diagnostic (real concurrent race,
artificially widened with a sleep) showed the losing connection instead
fails its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with
this driver, before ON CONFLICT DO NOTHING can ever report 0 rows affected
-- meaning the specific branch this bug lived in may be effectively
unreachable via this driver/journal-mode combination today. The shipped
test verifies the weaker, still real property that matters operationally:
many genuinely concurrent same-turn-ID claims across separate connections
never produce the bug's error signature ("sessionstore: read turn:"),
distinct from unrelated SQLITE_BUSY/SQLITE_BUSY_SNAPSHOT lock contention,
which is retried rather than treated as a failure. The fix stands
regardless of provable live reachability -- the shadowing was unambiguous
and wrong on its own terms.

Adversarially reviewed by a fresh subagent, which independently reverted
just the fix (keeping the test) and confirmed empirically -- 20 runs, plus
1000 trials with a debug probe on the affected!=1 branch -- that the branch
is never reached under this driver, corroborating the test's own stated
limitation rather than contradicting it. gocognit/nestif findings on
ClaimTurn are pre-existing on the prior committed baseline, unchanged by
this fix (only variable names moved).
This commit is contained in:
sam committed 2026-08-22 02:03:27 +10:00
1 parent 0ae6440a66
commit d3f9a7a472
2 files changed
+157 -7

No files matched your search

+17 -7
View File
@@ -224,19 +224,27 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
initial.CreatedAt = now
}
initial.UpdatedAt = now
result, err := tx.ExecContext(ctx, `
// insertErr is deliberately its own name, not err: this whole block
// is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here
// would shadow the outer err for the rest of the block -- including
// the retry read below, whose result must reach the outer `if err
// != nil` after this block closes. A shadowed reassignment there
// would go out of scope silently, leaving the outer err at its
// original sql.ErrNoRows value regardless of how the retry read
// actually went.
result, insertErr := tx.ExecContext(ctx, `
INSERT INTO turns (
turn_id, session_id, source_id, status, attempt, owner_id,
input_message_id, assistant_message_id, partial_text,
response_text, error, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...)
if err != nil {
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", err)
if insertErr != nil {
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr)
}
affected, err := result.RowsAffected()
if err != nil {
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", err)
affected, insertErr := result.RowsAffected()
if insertErr != nil {
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", insertErr)
}
if affected == 1 {
if err := tx.Commit(); err != nil {
@@ -246,7 +254,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
}
// Another connection inserted the same turn between our read and
// insert. Read it in this transaction and apply the normal claim
// state machine instead of surfacing a uniqueness error.
// state machine instead of surfacing a uniqueness error. This
// assigns the OUTER err (see above) -- it is what the `if err !=
// nil` check below this block actually sees.
err = tx.QueryRowContext(ctx, `
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
input_message_id, assistant_message_id, partial_text,
@@ -0,0 +1,140 @@
package gateway
import (
"context"
"fmt"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// TestClaimTurnConcurrentInsertRaceNeverReturnsAnError guards against a
// variable-shadowing bug in ClaimTurn (session_store_sqlite.go), fixed
// alongside this test: the branch handling "another connection inserted the
// same turn between our read and insert" used to write its retry read's
// result into an `err` shadowed inside the `errors.Is(err, sql.ErrNoRows)`
// block by an earlier `result, err := tx.ExecContext(...)`. That shadowed
// err went out of scope at the block's closing brace, so the outer err
// checked afterward was still sql.ErrNoRows from the original read -- the
// loser of that race would always return an error wrapped
// "sessionstore: read turn: %w", even though the comment's own stated
// intent is "apply the normal claim state machine instead of surfacing a
// uniqueness error." Confirmed via careful reading of Go's block-scoping
// rules, not assumed.
//
// Honest limitation: this test could NOT be made to reproduce that exact
// branch through genuine concurrent access. A one-off diagnostic (real
// two-connection race, sleep-widened to force overlap, reverted before this
// commit -- never present in a shipped file) showed the loser instead fails
// its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with this
// driver, before ON CONFLICT DO NOTHING can ever report affected=0. That
// means the specific "0 rows affected" branch this bug lived in may be
// effectively unreachable via a same-driver, same-journal-mode race today.
// The fix stands regardless -- the shadowing was real, wrong on its own
// terms, and free to correct -- but this test cannot claim to exercise the
// exact line that was fixed. What it verifies is the weaker, still real
// property that matters operationally: many genuinely concurrent
// same-turn-ID claims across separate connections never produce the bug's
// specific error signature. Treat this as a guard against a regression
// becoming reachable (a different SQLite version, journal mode, or driver
// with weaker snapshot isolation), not as proof the original bug was ever
// live in production.
//
// A single sqliteSessionStore instance cannot exhibit any of this: its own
// mutex serializes every ClaimTurn call against that instance's connection,
// so this uses two separate store instances (two connections, two mutexes)
// against the same file-backed database -- the real-world shape of the
// scenario (two archied/worker processes or connections sharing one session
// DB).
//
// SQLite's own write-lock contention (SQLITE_BUSY / SQLITE_BUSY_SNAPSHOT,
// "database is locked") is infrastructure noise distinct from the bug: it
// comes from BeginTx/ExecContext/Commit failing outright and is wrapped
// with a different prefix ("claim turn: begin/insert/rows affected/commit").
// Only the "sessionstore: read turn:" wrap was the bug's unique signature,
// so transient busy errors are retried rather than treated as failures.
func TestClaimTurnConcurrentInsertRaceNeverReturnsAnError(t *testing.T) {
path := filepath.Join(t.TempDir(), "sessions.db")
const stores = 2
const trials = 100
const maxRetriesPerAttempt = 20
openers := make([]TurnLedger, stores)
for i := range openers {
store, err := OpenSQLiteSessionStore(path)
if err != nil {
t.Fatalf("OpenSQLiteSessionStore[%d]: %v", i, err)
}
closer, ok := store.(interface{ Close() error })
if !ok {
t.Fatalf("store[%d] is %T, want a Close() error", i, store)
}
t.Cleanup(func() { _ = closer.Close() })
ledger, ok := store.(TurnLedger)
if !ok {
t.Fatalf("store[%d] is %T, want TurnLedger", i, store)
}
openers[i] = ledger
}
ctx := context.Background()
now := time.UnixMilli(1000).UTC()
claimRetryingBusy := func(ledger TurnLedger, initial TurnRecord) error {
var err error
for range maxRetriesPerAttempt {
_, _, err = ledger.ClaimTurn(ctx, initial)
if err == nil || !strings.Contains(err.Error(), "database is locked") {
return err
}
}
return err
}
// All trials race concurrently, not one at a time: a sequential
// trial-by-trial race tends to resolve cleanly before the next trial
// starts (the loser's retry sees an already-committed winner and skips
// the buggy branch entirely), which undersells how likely the narrow
// timing window is under real, overlapping load.
errs := make([][]error, trials)
var wg sync.WaitGroup
var start sync.WaitGroup
start.Add(1)
for trial := range trials {
turnID := fmt.Sprintf("race-turn-%d", trial)
initial := TurnRecord{
TurnID: turnID,
SessionID: "race-session",
SourceID: turnID,
Status: TurnStatusAccepted,
CreatedAt: now,
UpdatedAt: now,
}
errs[trial] = make([]error, stores)
for i := range stores {
wg.Add(1)
go func(trial, i int) {
defer wg.Done()
start.Wait()
errs[trial][i] = claimRetryingBusy(openers[i], initial)
}(trial, i)
}
}
start.Done()
wg.Wait()
for trial, storeErrs := range errs {
for i, err := range storeErrs {
if err == nil {
continue
}
if strings.Contains(err.Error(), "sessionstore: read turn:") {
t.Fatalf("trial %d, store %d: ClaimTurn() error = %v -- a losing racer surfaced the winner's insert as a read failure instead of applying the normal claim state machine", trial, i, err)
}
t.Fatalf("trial %d, store %d: ClaimTurn() unexpected error = %v", trial, i, err)
}
}
}