fix(gateway): stop ClaimTurn's insert-race branch losing its own retry read
Found while investigating an `ineffassign` finding, not a stylistic report:
the branch handling "another connection inserted the same turn between our
read and insert" was discarding its own result.
`result, err := tx.ExecContext(ctx, ...)` for the INSERT sits inside
`if errors.Is(err, sql.ErrNoRows) { ... }`, a new block scope, with `result`
new to that scope -- so Go's `:=` declares a fresh, block-scoped `err`
shadowing the function's own `err` for the rest of the block, including the
retry read a few lines later (`err = tx.QueryRowContext(...).Scan(...)`).
That assignment landed in the shadow, which goes out of scope at the block's
closing brace. The `if err != nil` check afterward was therefore still
looking at the *original* err -- sql.ErrNoRows from the very first read --
regardless of whether the retry read succeeded. The loser of a genuine
insert race would always return `sessionstore: read turn: %w` wrapping
`sql.ErrNoRows`, contradicting the branch's own comment: "apply the normal
claim state machine instead of surfacing a uniqueness error."
Fix: the insert-path errors are now insertErr, not err, so the retry read's
plain assignment reaches the outer err the later check actually inspects.
Confirmed via careful trace of Go's block-scoping rules that this closes the
gap for the affected!=1 (loser) path without touching the affected==1
(winner) path, which returns from its own separately-scoped err before ever
reaching this one.
Commit the honest limitation along with the fix: a genuine two-connection
concurrent regression test could not be made to hit the exact branch. A
temporary, reverted-before-commit diagnostic (real concurrent race,
artificially widened with a sleep) showed the losing connection instead
fails its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with
this driver, before ON CONFLICT DO NOTHING can ever report 0 rows affected
-- meaning the specific branch this bug lived in may be effectively
unreachable via this driver/journal-mode combination today. The shipped
test verifies the weaker, still real property that matters operationally:
many genuinely concurrent same-turn-ID claims across separate connections
never produce the bug's error signature ("sessionstore: read turn:"),
distinct from unrelated SQLITE_BUSY/SQLITE_BUSY_SNAPSHOT lock contention,
which is retried rather than treated as a failure. The fix stands
regardless of provable live reachability -- the shadowing was unambiguous
and wrong on its own terms.
Adversarially reviewed by a fresh subagent, which independently reverted
just the fix (keeping the test) and confirmed empirically -- 20 runs, plus
1000 trials with a debug probe on the affected!=1 branch -- that the branch
is never reached under this driver, corroborating the test's own stated
limitation rather than contradicting it. gocognit/nestif findings on
ClaimTurn are pre-existing on the prior committed baseline, unchanged by
this fix (only variable names moved).
This commit is contained in:
1 parent
a0d569da09
commit
311f5f71f9
2 files changed
+157
-7
No files matched your search
@@ -224,19 +224,27 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
initial.CreatedAt = now
|
||||
}
|
||||
initial.UpdatedAt = now
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
// insertErr is deliberately its own name, not err: this whole block
|
||||
// is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here
|
||||
// would shadow the outer err for the rest of the block -- including
|
||||
// the retry read below, whose result must reach the outer `if err
|
||||
// != nil` after this block closes. A shadowed reassignment there
|
||||
// would go out of scope silently, leaving the outer err at its
|
||||
// original sql.ErrNoRows value regardless of how the retry read
|
||||
// actually went.
|
||||
result, insertErr := tx.ExecContext(ctx, `
|
||||
INSERT INTO turns (
|
||||
turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...)
|
||||
if err != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", err)
|
||||
if insertErr != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr)
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", err)
|
||||
affected, insertErr := result.RowsAffected()
|
||||
if insertErr != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: rows affected: %w", insertErr)
|
||||
}
|
||||
if affected == 1 {
|
||||
if err := tx.Commit(); err != nil {
|
||||
@@ -246,7 +254,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
}
|
||||
// Another connection inserted the same turn between our read and
|
||||
// insert. Read it in this transaction and apply the normal claim
|
||||
// state machine instead of surfacing a uniqueness error.
|
||||
// state machine instead of surfacing a uniqueness error. This
|
||||
// assigns the OUTER err (see above) -- it is what the `if err !=
|
||||
// nil` check below this block actually sees.
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestClaimTurnConcurrentInsertRaceNeverReturnsAnError guards against a
|
||||
// variable-shadowing bug in ClaimTurn (session_store_sqlite.go), fixed
|
||||
// alongside this test: the branch handling "another connection inserted the
|
||||
// same turn between our read and insert" used to write its retry read's
|
||||
// result into an `err` shadowed inside the `errors.Is(err, sql.ErrNoRows)`
|
||||
// block by an earlier `result, err := tx.ExecContext(...)`. That shadowed
|
||||
// err went out of scope at the block's closing brace, so the outer err
|
||||
// checked afterward was still sql.ErrNoRows from the original read -- the
|
||||
// loser of that race would always return an error wrapped
|
||||
// "sessionstore: read turn: %w", even though the comment's own stated
|
||||
// intent is "apply the normal claim state machine instead of surfacing a
|
||||
// uniqueness error." Confirmed via careful reading of Go's block-scoping
|
||||
// rules, not assumed.
|
||||
//
|
||||
// Honest limitation: this test could NOT be made to reproduce that exact
|
||||
// branch through genuine concurrent access. A one-off diagnostic (real
|
||||
// two-connection race, sleep-widened to force overlap, reverted before this
|
||||
// commit -- never present in a shipped file) showed the loser instead fails
|
||||
// its own INSERT outright with SQLITE_BUSY_SNAPSHOT under WAL mode with this
|
||||
// driver, before ON CONFLICT DO NOTHING can ever report affected=0. That
|
||||
// means the specific "0 rows affected" branch this bug lived in may be
|
||||
// effectively unreachable via a same-driver, same-journal-mode race today.
|
||||
// The fix stands regardless -- the shadowing was real, wrong on its own
|
||||
// terms, and free to correct -- but this test cannot claim to exercise the
|
||||
// exact line that was fixed. What it verifies is the weaker, still real
|
||||
// property that matters operationally: many genuinely concurrent
|
||||
// same-turn-ID claims across separate connections never produce the bug's
|
||||
// specific error signature. Treat this as a guard against a regression
|
||||
// becoming reachable (a different SQLite version, journal mode, or driver
|
||||
// with weaker snapshot isolation), not as proof the original bug was ever
|
||||
// live in production.
|
||||
//
|
||||
// A single sqliteSessionStore instance cannot exhibit any of this: its own
|
||||
// mutex serializes every ClaimTurn call against that instance's connection,
|
||||
// so this uses two separate store instances (two connections, two mutexes)
|
||||
// against the same file-backed database -- the real-world shape of the
|
||||
// scenario (two archied/worker processes or connections sharing one session
|
||||
// DB).
|
||||
//
|
||||
// SQLite's own write-lock contention (SQLITE_BUSY / SQLITE_BUSY_SNAPSHOT,
|
||||
// "database is locked") is infrastructure noise distinct from the bug: it
|
||||
// comes from BeginTx/ExecContext/Commit failing outright and is wrapped
|
||||
// with a different prefix ("claim turn: begin/insert/rows affected/commit").
|
||||
// Only the "sessionstore: read turn:" wrap was the bug's unique signature,
|
||||
// so transient busy errors are retried rather than treated as failures.
|
||||
func TestClaimTurnConcurrentInsertRaceNeverReturnsAnError(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
|
||||
const stores = 2
|
||||
const trials = 100
|
||||
const maxRetriesPerAttempt = 20
|
||||
|
||||
openers := make([]TurnLedger, stores)
|
||||
for i := range openers {
|
||||
store, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore[%d]: %v", i, err)
|
||||
}
|
||||
closer, ok := store.(interface{ Close() error })
|
||||
if !ok {
|
||||
t.Fatalf("store[%d] is %T, want a Close() error", i, store)
|
||||
}
|
||||
t.Cleanup(func() { _ = closer.Close() })
|
||||
ledger, ok := store.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("store[%d] is %T, want TurnLedger", i, store)
|
||||
}
|
||||
openers[i] = ledger
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
now := time.UnixMilli(1000).UTC()
|
||||
|
||||
claimRetryingBusy := func(ledger TurnLedger, initial TurnRecord) error {
|
||||
var err error
|
||||
for range maxRetriesPerAttempt {
|
||||
_, _, err = ledger.ClaimTurn(ctx, initial)
|
||||
if err == nil || !strings.Contains(err.Error(), "database is locked") {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// All trials race concurrently, not one at a time: a sequential
|
||||
// trial-by-trial race tends to resolve cleanly before the next trial
|
||||
// starts (the loser's retry sees an already-committed winner and skips
|
||||
// the buggy branch entirely), which undersells how likely the narrow
|
||||
// timing window is under real, overlapping load.
|
||||
errs := make([][]error, trials)
|
||||
var wg sync.WaitGroup
|
||||
var start sync.WaitGroup
|
||||
start.Add(1)
|
||||
for trial := range trials {
|
||||
turnID := fmt.Sprintf("race-turn-%d", trial)
|
||||
initial := TurnRecord{
|
||||
TurnID: turnID,
|
||||
SessionID: "race-session",
|
||||
SourceID: turnID,
|
||||
Status: TurnStatusAccepted,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
errs[trial] = make([]error, stores)
|
||||
for i := range stores {
|
||||
wg.Add(1)
|
||||
go func(trial, i int) {
|
||||
defer wg.Done()
|
||||
start.Wait()
|
||||
errs[trial][i] = claimRetryingBusy(openers[i], initial)
|
||||
}(trial, i)
|
||||
}
|
||||
}
|
||||
start.Done()
|
||||
wg.Wait()
|
||||
|
||||
for trial, storeErrs := range errs {
|
||||
for i, err := range storeErrs {
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(err.Error(), "sessionstore: read turn:") {
|
||||
t.Fatalf("trial %d, store %d: ClaimTurn() error = %v -- a losing racer surfaced the winner's insert as a read failure instead of applying the normal claim state machine", trial, i, err)
|
||||
}
|
||||
t.Fatalf("trial %d, store %d: ClaimTurn() unexpected error = %v", trial, i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user