Merge pull request #559 from samcharles93/worktree-reflective-cuddling-salamander
fix(gateway): replay completed-duplicate turns with their tool activity
This commit is contained in:
commit
f76645bc2c
7 files changed
+340
-21
No files matched your search
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -102,6 +103,7 @@ CREATE TABLE IF NOT EXISTS turns (
|
||||
assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '',
|
||||
response_text TEXT NOT NULL DEFAULT '',
|
||||
tool_calls TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
@@ -151,6 +153,15 @@ func openSQLiteSessionStore(dsn string) (SessionStore, error) {
|
||||
if _, err := db.ExecContext(ctx, sqliteSessionSchema); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("sessionstore: init schema: %w", err), db.Close())
|
||||
}
|
||||
// A database created before tool_calls existed has a turns table without
|
||||
// it; CREATE TABLE IF NOT EXISTS above leaves an existing table alone, so
|
||||
// the column is added here. The duplicate-column error is the documented
|
||||
// way to detect "already migrated" with this driver -- there is no
|
||||
// IF NOT EXISTS clause for ADD COLUMN.
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE turns ADD COLUMN tool_calls TEXT NOT NULL DEFAULT '[]'`); err != nil &&
|
||||
!strings.Contains(err.Error(), "duplicate column name") {
|
||||
return nil, errors.Join(fmt.Errorf("sessionstore: migrate turns table: %w", err), db.Close())
|
||||
}
|
||||
if err := repairFutureMessageTimestamps(ctx, db, time.Now().UTC()); err != nil {
|
||||
return nil, errors.Join(err, db.Close())
|
||||
}
|
||||
@@ -207,14 +218,15 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, initial.TurnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
now := time.Now().UTC()
|
||||
@@ -224,6 +236,10 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
initial.CreatedAt = now
|
||||
}
|
||||
initial.UpdatedAt = now
|
||||
initialToolCalls, marshalErr := marshalToolCalls(initial.ToolCalls)
|
||||
if marshalErr != nil {
|
||||
return TurnRecord{}, "", marshalErr
|
||||
}
|
||||
// insertErr is deliberately its own name, not err: this whole block
|
||||
// is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here
|
||||
// would shadow the outer err for the rest of the block -- including
|
||||
@@ -236,9 +252,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
INSERT INTO turns (
|
||||
turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...)
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial, initialToolCalls)...)
|
||||
if insertErr != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr)
|
||||
}
|
||||
@@ -260,11 +276,11 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, initial.TurnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -272,6 +288,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return TurnRecord{}, "", err
|
||||
}
|
||||
|
||||
switch turn.Status {
|
||||
case TurnStatusCompleted:
|
||||
@@ -348,14 +367,15 @@ func (s *sqliteSessionStore) GetTurn(ctx context.Context, turnID string) (TurnRe
|
||||
defer s.mu.Unlock()
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, turnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TurnRecord{}, false, nil
|
||||
@@ -365,6 +385,9 @@ func (s *sqliteSessionStore) GetTurn(ctx context.Context, turnID string) (TurnRe
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return TurnRecord{}, false, err
|
||||
}
|
||||
return turn, true, nil
|
||||
}
|
||||
|
||||
@@ -389,15 +412,19 @@ func (s *sqliteSessionStore) SaveTurn(ctx context.Context, turn TurnRecord) erro
|
||||
if turn.UpdatedAt.IsZero() {
|
||||
turn.UpdatedAt = time.Now().UTC()
|
||||
}
|
||||
toolCallsJSON, err := marshalToolCalls(turn.ToolCalls)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `
|
||||
UPDATE turns SET
|
||||
session_id = ?, source_id = ?, status = ?, attempt = ?, owner_id = ?,
|
||||
input_message_id = ?, assistant_message_id = ?, partial_text = ?,
|
||||
response_text = ?, error = ?, created_at = ?, updated_at = ?
|
||||
response_text = ?, tool_calls = ?, error = ?, created_at = ?, updated_at = ?
|
||||
WHERE turn_id = ? AND attempt = ? AND owner_id = ?`,
|
||||
turn.SessionID, turn.SourceID, string(turn.Status), turn.Attempt, turn.OwnerID,
|
||||
turn.InputMessageID, turn.AssistantMessageID, turn.PartialText,
|
||||
turn.ResponseText, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.ResponseText, toolCallsJSON, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.TurnID, turn.Attempt, turn.OwnerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: save turn: %w", err)
|
||||
@@ -416,7 +443,7 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE status IN (?, ?, ?) ORDER BY updated_at ASC`,
|
||||
string(TurnStatusAccepted), string(TurnStatusRunning), string(TurnStatusPartial))
|
||||
if err != nil {
|
||||
@@ -427,15 +454,19 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
for rows.Next() {
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
if err := rows.Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan recoverable turn: %w", err)
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, turn)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -444,14 +475,42 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func turnValues(turn TurnRecord) []any {
|
||||
func turnValues(turn TurnRecord, toolCallsJSON string) []any {
|
||||
return []any{
|
||||
turn.TurnID, turn.SessionID, turn.SourceID, string(turn.Status), turn.Attempt, turn.OwnerID,
|
||||
turn.InputMessageID, turn.AssistantMessageID, turn.PartialText,
|
||||
turn.ResponseText, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.ResponseText, toolCallsJSON, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
}
|
||||
}
|
||||
|
||||
// marshalToolCalls encodes a turn's recorded tool activity for storage. An
|
||||
// empty slice becomes "[]" rather than SQL NULL or the JSON "null" so an
|
||||
// unmigrated-looking row (the column's own DEFAULT) and a turn that recorded
|
||||
// no tool calls are indistinguishable in the database, and both round-trip
|
||||
// through unmarshalToolCalls to a nil slice.
|
||||
func marshalToolCalls(events []ToolCallEvent) (string, error) {
|
||||
if len(events) == 0 {
|
||||
return "[]", nil
|
||||
}
|
||||
encoded, err := json.Marshal(events)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("sessionstore: marshal tool calls: %w", err)
|
||||
}
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
// unmarshalToolCalls decodes tool activity persisted by marshalToolCalls.
|
||||
func unmarshalToolCalls(raw string) ([]ToolCallEvent, error) {
|
||||
if raw == "" || raw == "[]" {
|
||||
return nil, nil
|
||||
}
|
||||
var events []ToolCallEvent
|
||||
if err := json.Unmarshal([]byte(raw), &events); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: unmarshal tool calls: %w", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// ── Sessions ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *sqliteSessionStore) Save(ctx context.Context, sc SessionContext) error {
|
||||
|
||||
@@ -25,6 +25,74 @@ func newTestSQLiteStore(t *testing.T) SessionStore {
|
||||
|
||||
func sqliteBase() time.Time { return time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
// TestOpenSQLiteSessionStoreMigratesPreExistingTurnsTable simulates a
|
||||
// database created before the tool_calls column existed: CREATE TABLE IF NOT
|
||||
// EXISTS leaves such a table alone, so opening it must add the column rather
|
||||
// than fail or silently keep the old shape.
|
||||
func TestOpenSQLiteSessionStoreMigratesPreExistingTurnsTable(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "legacy.db")
|
||||
raw, err := sql.Open("sqlite", sqliteSessionDSN(path))
|
||||
if err != nil {
|
||||
t.Fatalf("sql.Open() error = %v", err)
|
||||
}
|
||||
if _, err := raw.ExecContext(t.Context(), `CREATE TABLE turns (
|
||||
turn_id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 0,
|
||||
owner_id TEXT NOT NULL DEFAULT '',
|
||||
input_message_id TEXT NOT NULL DEFAULT '',
|
||||
assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '',
|
||||
response_text TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
t.Fatalf("create legacy turns table: %v", err)
|
||||
}
|
||||
if _, err := raw.ExecContext(t.Context(), `INSERT INTO turns (
|
||||
turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text, response_text,
|
||||
error, created_at, updated_at
|
||||
) VALUES ('legacy-turn', 'legacy-session', 'legacy-source', 'completed', 1, 'owner',
|
||||
'', 'assistant-1', '', 'legacy answer', '', 0, 0)`); err != nil {
|
||||
t.Fatalf("insert legacy turn row: %v", err)
|
||||
}
|
||||
if err := raw.Close(); err != nil {
|
||||
t.Fatalf("close legacy handle: %v", err)
|
||||
}
|
||||
|
||||
store, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore() on legacy schema error = %v", err)
|
||||
}
|
||||
defer func() { _ = store.Close() }()
|
||||
ledger, ok := store.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want TurnLedger", store)
|
||||
}
|
||||
turn, ok, err := ledger.GetTurn(t.Context(), "legacy-turn")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() after migration = %#v, %v, %v; want the legacy row", turn, ok, err)
|
||||
}
|
||||
if turn.ResponseText != "legacy answer" || len(turn.ToolCalls) != 0 {
|
||||
t.Fatalf("migrated turn = %#v, want legacy text with no tool calls", turn)
|
||||
}
|
||||
|
||||
// Reopening a second time must not fail on the "duplicate column name"
|
||||
// the migration produces once the column already exists.
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
reopened, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("second OpenSQLiteSessionStore() error = %v", err)
|
||||
}
|
||||
_ = reopened.Close()
|
||||
}
|
||||
|
||||
func TestSQLiteSessionStoreRejectsExcessiveSearchQueries(t *testing.T) {
|
||||
st := newTestSQLiteStore(t)
|
||||
tests := []struct {
|
||||
|
||||
@@ -147,6 +147,33 @@ func (f DeltaFunc) Delta(text string) {
|
||||
// ToolCall discards the event: a text-only caller has nowhere to put it.
|
||||
func (f DeltaFunc) ToolCall(ToolCallEvent) {}
|
||||
|
||||
// toolCallRecorder wraps a TurnStream, forwarding every event to it
|
||||
// unchanged while also recording the tool calls in arrival order. The
|
||||
// recording happens regardless of whether next is nil (a non-streaming
|
||||
// caller, e.g. Router.LLM) so a turn's tool activity is captured for
|
||||
// persistence and later replay even when nothing rendered it live the first
|
||||
// time.
|
||||
type toolCallRecorder struct {
|
||||
next TurnStream
|
||||
events []ToolCallEvent
|
||||
}
|
||||
|
||||
// Delta forwards the fragment to the wrapped stream, if any.
|
||||
func (r *toolCallRecorder) Delta(text string) {
|
||||
if r.next != nil {
|
||||
r.next.Delta(text)
|
||||
}
|
||||
}
|
||||
|
||||
// ToolCall records the event, then forwards it to the wrapped stream, if
|
||||
// any.
|
||||
func (r *toolCallRecorder) ToolCall(event ToolCallEvent) {
|
||||
r.events = append(r.events, event)
|
||||
if r.next != nil {
|
||||
r.next.ToolCall(event)
|
||||
}
|
||||
}
|
||||
|
||||
// firstNonEmptyLine returns the first line of s that holds something other
|
||||
// than whitespace, trimmed. It returns "" when there is no such line.
|
||||
func firstNonEmptyLine(s string) string {
|
||||
|
||||
@@ -125,8 +125,20 @@ func (r *TurnRunner) RespondStream(ctx context.Context, msg Message, stream Turn
|
||||
|
||||
// Run executes one turn, reporting progress to stream. A completed duplicate
|
||||
// source message replays its durable response without invoking the model
|
||||
// again; the replay reaches stream as one whole-text fragment, since there is
|
||||
// no generation left to watch.
|
||||
// again; the replay reaches stream as the turn's recorded tool-call events,
|
||||
// in the order they originally ran, followed by one whole-text fragment for
|
||||
// the answer, since there is no generation left to watch. Replaying the tool
|
||||
// events first is what keeps a redelivered or restart-recovered duplicate
|
||||
// from disagreeing with the original about what ran.
|
||||
//
|
||||
// The prior-reply fallback below (matching an already-answered source
|
||||
// message by conversation history rather than by turn ID) is deliberately
|
||||
// text-only: it exists for the narrow race where a process crashed after
|
||||
// saving the assistant message but before the turn record was marked
|
||||
// completed, so the original turn's structured tool record -- saved in the
|
||||
// same call as ResponseText -- never made it to durable storage either. It
|
||||
// has nothing to replay, so it must not synthesize a tool block it cannot
|
||||
// verify.
|
||||
//
|
||||
// stream may be nil, which means the caller renders only the returned reply.
|
||||
func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (string, error) {
|
||||
@@ -172,8 +184,13 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
}
|
||||
switch claim {
|
||||
case TurnClaimCompleted:
|
||||
if stream != nil && turn.ResponseText != "" {
|
||||
stream.Delta(turn.ResponseText)
|
||||
if stream != nil {
|
||||
for _, event := range turn.ToolCalls {
|
||||
stream.ToolCall(event)
|
||||
}
|
||||
if turn.ResponseText != "" {
|
||||
stream.Delta(turn.ResponseText)
|
||||
}
|
||||
}
|
||||
return turn.ResponseText, nil
|
||||
case TurnClaimInProgress:
|
||||
@@ -284,10 +301,11 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
view.Messages = append(
|
||||
[]CompressedMessage{{Role: "system", Content: systemPrompt}}, view.Messages...,
|
||||
)
|
||||
recorder := &toolCallRecorder{next: stream}
|
||||
text, err := prepared.Generate(ctx, TurnModelRequest{
|
||||
Messages: view.Messages,
|
||||
MaxTokens: modelDetails.MaxOutputTokens,
|
||||
}, stream)
|
||||
}, recorder)
|
||||
if err != nil {
|
||||
return "", r.failTurn(ctx, turn, err)
|
||||
}
|
||||
@@ -298,6 +316,7 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
turn.Status = TurnStatusCompleted
|
||||
turn.AssistantMessageID = assistant.MessageID
|
||||
turn.ResponseText = text
|
||||
turn.ToolCalls = recorder.events
|
||||
turn.Error = ""
|
||||
turn.UpdatedAt = time.Now().UTC()
|
||||
if err := r.saveTurn(ctx, turn); err != nil {
|
||||
|
||||
@@ -39,7 +39,11 @@ var ErrTurnOwnershipLost = errors.New("chat turn ownership lost")
|
||||
|
||||
// TurnRecord is the durable identity and outcome projection for one chat
|
||||
// generation. ResponseText is retained so a completed duplicate can replay
|
||||
// without depending on the bounded model-context history window.
|
||||
// without depending on the bounded model-context history window. ToolCalls
|
||||
// is retained for the same reason and saved in the same call as
|
||||
// ResponseText, so a completed-duplicate replay (NATS redelivery, restart
|
||||
// RecoverTurns redelivery) can narrate the tool activity that produced the
|
||||
// answer instead of showing a tool-less duplicate beside the original.
|
||||
type TurnRecord struct {
|
||||
TurnID string
|
||||
SessionID string
|
||||
@@ -51,6 +55,7 @@ type TurnRecord struct {
|
||||
AssistantMessageID string
|
||||
PartialText string
|
||||
ResponseText string
|
||||
ToolCalls []ToolCallEvent
|
||||
Error string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
|
||||
@@ -207,6 +207,68 @@ func TestSessionDeleteRemovesTurns(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTurnLedgerPersistsToolCallsAcrossClaimAndReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
ctx := context.Background()
|
||||
store, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore() error = %v", err)
|
||||
}
|
||||
ledger, ok := store.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want TurnLedger", store)
|
||||
}
|
||||
turn := TurnRecord{
|
||||
TurnID: CanonicalTurnID("session-tools", "source-tools"),
|
||||
SessionID: "session-tools",
|
||||
SourceID: "source-tools",
|
||||
OwnerID: "process-a",
|
||||
Status: TurnStatusAccepted,
|
||||
}
|
||||
claimed, claim, err := ledger.ClaimTurn(ctx, turn)
|
||||
if err != nil || claim != TurnClaimOwned {
|
||||
t.Fatalf("ClaimTurn() = %#v, %q, %v", claimed, claim, err)
|
||||
}
|
||||
claimed.Status = TurnStatusCompleted
|
||||
claimed.ResponseText = "answer"
|
||||
claimed.AssistantMessageID = "assistant-1"
|
||||
claimed.ToolCalls = []ToolCallEvent{
|
||||
{ID: "call-1", Name: "shell", Parameters: `{"cmd":"ls"}`, Output: "a.txt"},
|
||||
{ID: "call-2", Name: "grep", Err: "no matches"},
|
||||
}
|
||||
if err := ledger.SaveTurn(ctx, claimed); err != nil {
|
||||
t.Fatalf("SaveTurn() error = %v", err)
|
||||
}
|
||||
|
||||
completed, claim, err := ledger.ClaimTurn(ctx, turn)
|
||||
if err != nil || claim != TurnClaimCompleted {
|
||||
t.Fatalf("re-claim = %#v, %q, %v", completed, claim, err)
|
||||
}
|
||||
if len(completed.ToolCalls) != 2 || completed.ToolCalls[0].Name != "shell" || completed.ToolCalls[1].Err != "no matches" {
|
||||
t.Fatalf("re-claimed tool calls = %#v, want the two saved events in order", completed.ToolCalls)
|
||||
}
|
||||
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
reopened, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen error = %v", err)
|
||||
}
|
||||
defer func() { _ = reopened.Close() }()
|
||||
reopenedLedger, ok := reopened.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("reopened store is %T, want TurnLedger", reopened)
|
||||
}
|
||||
got, ok, err := reopenedLedger.GetTurn(ctx, turn.TurnID)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() after reopen = %#v, %v, %v", got, ok, err)
|
||||
}
|
||||
if len(got.ToolCalls) != 2 || got.ToolCalls[0].Name != "shell" || got.ToolCalls[1].Name != "grep" {
|
||||
t.Fatalf("reopened tool calls = %#v, want the two saved events in order", got.ToolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteTurnLedgerSurvivesReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -99,6 +99,85 @@ func TestTurnRunnerReplayStreamsStoredReplyOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A completed-duplicate replay (NATS redelivery, restart RecoverTurns
|
||||
// redelivery) must narrate the same tool activity the original turn did, in
|
||||
// the same order, ahead of the answer -- otherwise the duplicate disagrees
|
||||
// with the original about what ran (archie-core-ippu).
|
||||
func TestTurnRunnerReplayCompletedDuplicateReplaysToolCalls(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "done"}
|
||||
prepared.generateStream = func(stream TurnStream) (string, error) {
|
||||
stream.ToolCall(ToolCallEvent{Name: "shell", Output: "exit 0"})
|
||||
stream.Delta("done")
|
||||
return "done", nil
|
||||
}
|
||||
runner := newStreamTestRunner(t, prepared)
|
||||
msg := Message{From: "user", ChannelID: "chat-1", SourceID: "source-1", Text: "hello"}
|
||||
|
||||
if _, err := runner.Run(context.Background(), msg, &recordingStream{}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
|
||||
sink := &recordingStream{}
|
||||
reply, err := runner.Run(context.Background(), msg, sink)
|
||||
if err != nil {
|
||||
t.Fatalf("replay Run() error = %v", err)
|
||||
}
|
||||
if reply != "done" {
|
||||
t.Fatalf("replayed reply = %q", reply)
|
||||
}
|
||||
want := []string{"tool:shell:exit 0", "text:done"}
|
||||
if len(sink.events) != len(want) {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
for i := range want {
|
||||
if sink.events[i] != want[i] {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
}
|
||||
if prepared.generateN != 1 {
|
||||
t.Fatalf("Generate calls = %d, want 1 -- a replay must not call the model", prepared.generateN)
|
||||
}
|
||||
|
||||
turn, ok, err := runner.Ledger.GetTurn(context.Background(), CanonicalTurnID("chat-1", "source-1"))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() = %#v, %v, %v; want persisted turn", turn, ok, err)
|
||||
}
|
||||
if len(turn.ToolCalls) != 1 || turn.ToolCalls[0].Name != "shell" {
|
||||
t.Fatalf("persisted turn tool calls = %#v, want one recorded shell call", turn.ToolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// A turn's recorded tool activity is captured even when the original caller
|
||||
// did not stream (Router.LLM, a nil sink): a later duplicate delivered to a
|
||||
// streaming caller must still be able to replay it.
|
||||
func TestTurnRunnerRecordsToolCallsEvenWithoutOriginalStream(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "done"}
|
||||
prepared.generateStream = func(stream TurnStream) (string, error) {
|
||||
stream.ToolCall(ToolCallEvent{Name: "grep", Output: "3 matches"})
|
||||
return "done", nil
|
||||
}
|
||||
runner := newStreamTestRunner(t, prepared)
|
||||
msg := Message{From: "user", ChannelID: "chat-1", SourceID: "source-1", Text: "hello"}
|
||||
|
||||
if _, err := runner.Run(context.Background(), msg, nil); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
|
||||
sink := &recordingStream{}
|
||||
if _, err := runner.Run(context.Background(), msg, sink); err != nil {
|
||||
t.Fatalf("replay Run() error = %v", err)
|
||||
}
|
||||
want := []string{"tool:grep:3 matches", "text:done"}
|
||||
if len(sink.events) != len(want) {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
for i := range want {
|
||||
if sink.events[i] != want[i] {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A nil sink means "not streaming": the turn still runs and returns its reply.
|
||||
func TestTurnRunnerAcceptsNoStream(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "quiet answer"}
|
||||
|
||||
Reference in new issue
Block a user