fix(gateway): replay completed-duplicate turns with their tool activity
A NATS redelivery or restart RecoverTurns redelivery hit the TurnClaimCompleted branch in TurnRunner.Run and replayed only turn.ResponseText, never the tool calls that produced it -- neither the ledger nor the session store persisted them. The transcript showed the original answer beside a tool-less duplicate, disagreeing about what ran. TurnRecord now carries ToolCalls, saved in the same call as ResponseText via a toolCallRecorder that tees Generate's TurnStream. The completed-duplicate replay walks turn.ToolCalls into stream.ToolCall before replaying the text, so a redelivered turn narrates the same activity as the original. The other replay path (prior-reply, matched by conversation history rather than turn ID) stays deliberately text-only: it exists for the crash race where the assistant message was saved but the turn record was never completed, so there is no persisted tool record for it to replay either. Documented on Run's doc comment rather than left implicit. Closes archie-core-ippu.
This commit is contained in:
1 parent
c7b2c2bf46
commit
bc0c6108fc
7 files changed
+340
-21
No files matched your search
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -102,6 +103,7 @@ CREATE TABLE IF NOT EXISTS turns (
|
||||
assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '',
|
||||
response_text TEXT NOT NULL DEFAULT '',
|
||||
tool_calls TEXT NOT NULL DEFAULT '[]',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
@@ -151,6 +153,15 @@ func openSQLiteSessionStore(dsn string) (SessionStore, error) {
|
||||
if _, err := db.ExecContext(ctx, sqliteSessionSchema); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("sessionstore: init schema: %w", err), db.Close())
|
||||
}
|
||||
// A database created before tool_calls existed has a turns table without
|
||||
// it; CREATE TABLE IF NOT EXISTS above leaves an existing table alone, so
|
||||
// the column is added here. The duplicate-column error is the documented
|
||||
// way to detect "already migrated" with this driver -- there is no
|
||||
// IF NOT EXISTS clause for ADD COLUMN.
|
||||
if _, err := db.ExecContext(ctx, `ALTER TABLE turns ADD COLUMN tool_calls TEXT NOT NULL DEFAULT '[]'`); err != nil &&
|
||||
!strings.Contains(err.Error(), "duplicate column name") {
|
||||
return nil, errors.Join(fmt.Errorf("sessionstore: migrate turns table: %w", err), db.Close())
|
||||
}
|
||||
if err := repairFutureMessageTimestamps(ctx, db, time.Now().UTC()); err != nil {
|
||||
return nil, errors.Join(err, db.Close())
|
||||
}
|
||||
@@ -207,14 +218,15 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, initial.TurnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
now := time.Now().UTC()
|
||||
@@ -224,6 +236,10 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
initial.CreatedAt = now
|
||||
}
|
||||
initial.UpdatedAt = now
|
||||
initialToolCalls, marshalErr := marshalToolCalls(initial.ToolCalls)
|
||||
if marshalErr != nil {
|
||||
return TurnRecord{}, "", marshalErr
|
||||
}
|
||||
// insertErr is deliberately its own name, not err: this whole block
|
||||
// is inside `if errors.Is(err, sql.ErrNoRows)`, so reusing err here
|
||||
// would shadow the outer err for the rest of the block -- including
|
||||
@@ -236,9 +252,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
INSERT INTO turns (
|
||||
turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial)...)
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(turn_id) DO NOTHING`, turnValues(initial, initialToolCalls)...)
|
||||
if insertErr != nil {
|
||||
return TurnRecord{}, "", fmt.Errorf("sessionstore: claim turn: insert: %w", insertErr)
|
||||
}
|
||||
@@ -260,11 +276,11 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
err = tx.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, initial.TurnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -272,6 +288,9 @@ func (s *sqliteSessionStore) ClaimTurn(ctx context.Context, initial TurnRecord)
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return TurnRecord{}, "", err
|
||||
}
|
||||
|
||||
switch turn.Status {
|
||||
case TurnStatusCompleted:
|
||||
@@ -348,14 +367,15 @@ func (s *sqliteSessionStore) GetTurn(ctx context.Context, turnID string) (TurnRe
|
||||
defer s.mu.Unlock()
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE turn_id = ?`, turnID).Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TurnRecord{}, false, nil
|
||||
@@ -365,6 +385,9 @@ func (s *sqliteSessionStore) GetTurn(ctx context.Context, turnID string) (TurnRe
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return TurnRecord{}, false, err
|
||||
}
|
||||
return turn, true, nil
|
||||
}
|
||||
|
||||
@@ -389,15 +412,19 @@ func (s *sqliteSessionStore) SaveTurn(ctx context.Context, turn TurnRecord) erro
|
||||
if turn.UpdatedAt.IsZero() {
|
||||
turn.UpdatedAt = time.Now().UTC()
|
||||
}
|
||||
toolCallsJSON, err := marshalToolCalls(turn.ToolCalls)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, `
|
||||
UPDATE turns SET
|
||||
session_id = ?, source_id = ?, status = ?, attempt = ?, owner_id = ?,
|
||||
input_message_id = ?, assistant_message_id = ?, partial_text = ?,
|
||||
response_text = ?, error = ?, created_at = ?, updated_at = ?
|
||||
response_text = ?, tool_calls = ?, error = ?, created_at = ?, updated_at = ?
|
||||
WHERE turn_id = ? AND attempt = ? AND owner_id = ?`,
|
||||
turn.SessionID, turn.SourceID, string(turn.Status), turn.Attempt, turn.OwnerID,
|
||||
turn.InputMessageID, turn.AssistantMessageID, turn.PartialText,
|
||||
turn.ResponseText, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.ResponseText, toolCallsJSON, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.TurnID, turn.Attempt, turn.OwnerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: save turn: %w", err)
|
||||
@@ -416,7 +443,7 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text,
|
||||
response_text, error, created_at, updated_at
|
||||
response_text, tool_calls, error, created_at, updated_at
|
||||
FROM turns WHERE status IN (?, ?, ?) ORDER BY updated_at ASC`,
|
||||
string(TurnStatusAccepted), string(TurnStatusRunning), string(TurnStatusPartial))
|
||||
if err != nil {
|
||||
@@ -427,15 +454,19 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
for rows.Next() {
|
||||
var turn TurnRecord
|
||||
var createdMS, updatedMS int64
|
||||
var toolCallsRaw string
|
||||
if err := rows.Scan(
|
||||
&turn.TurnID, &turn.SessionID, &turn.SourceID, &turn.Status, &turn.Attempt, &turn.OwnerID,
|
||||
&turn.InputMessageID, &turn.AssistantMessageID, &turn.PartialText,
|
||||
&turn.ResponseText, &turn.Error, &createdMS, &updatedMS,
|
||||
&turn.ResponseText, &toolCallsRaw, &turn.Error, &createdMS, &updatedMS,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan recoverable turn: %w", err)
|
||||
}
|
||||
turn.CreatedAt = time.UnixMilli(createdMS).UTC()
|
||||
turn.UpdatedAt = time.UnixMilli(updatedMS).UTC()
|
||||
if turn.ToolCalls, err = unmarshalToolCalls(toolCallsRaw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, turn)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -444,14 +475,42 @@ func (s *sqliteSessionStore) ListRecoverableTurns(ctx context.Context) ([]TurnRe
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func turnValues(turn TurnRecord) []any {
|
||||
func turnValues(turn TurnRecord, toolCallsJSON string) []any {
|
||||
return []any{
|
||||
turn.TurnID, turn.SessionID, turn.SourceID, string(turn.Status), turn.Attempt, turn.OwnerID,
|
||||
turn.InputMessageID, turn.AssistantMessageID, turn.PartialText,
|
||||
turn.ResponseText, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
turn.ResponseText, toolCallsJSON, turn.Error, turn.CreatedAt.UnixMilli(), turn.UpdatedAt.UnixMilli(),
|
||||
}
|
||||
}
|
||||
|
||||
// marshalToolCalls encodes a turn's recorded tool activity for storage. An
|
||||
// empty slice becomes "[]" rather than SQL NULL or the JSON "null" so an
|
||||
// unmigrated-looking row (the column's own DEFAULT) and a turn that recorded
|
||||
// no tool calls are indistinguishable in the database, and both round-trip
|
||||
// through unmarshalToolCalls to a nil slice.
|
||||
func marshalToolCalls(events []ToolCallEvent) (string, error) {
|
||||
if len(events) == 0 {
|
||||
return "[]", nil
|
||||
}
|
||||
encoded, err := json.Marshal(events)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("sessionstore: marshal tool calls: %w", err)
|
||||
}
|
||||
return string(encoded), nil
|
||||
}
|
||||
|
||||
// unmarshalToolCalls decodes tool activity persisted by marshalToolCalls.
|
||||
func unmarshalToolCalls(raw string) ([]ToolCallEvent, error) {
|
||||
if raw == "" || raw == "[]" {
|
||||
return nil, nil
|
||||
}
|
||||
var events []ToolCallEvent
|
||||
if err := json.Unmarshal([]byte(raw), &events); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: unmarshal tool calls: %w", err)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// ── Sessions ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *sqliteSessionStore) Save(ctx context.Context, sc SessionContext) error {
|
||||
|
||||
@@ -25,6 +25,74 @@ func newTestSQLiteStore(t *testing.T) SessionStore {
|
||||
|
||||
func sqliteBase() time.Time { return time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
// TestOpenSQLiteSessionStoreMigratesPreExistingTurnsTable simulates a
|
||||
// database created before the tool_calls column existed: CREATE TABLE IF NOT
|
||||
// EXISTS leaves such a table alone, so opening it must add the column rather
|
||||
// than fail or silently keep the old shape.
|
||||
func TestOpenSQLiteSessionStoreMigratesPreExistingTurnsTable(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "legacy.db")
|
||||
raw, err := sql.Open("sqlite", sqliteSessionDSN(path))
|
||||
if err != nil {
|
||||
t.Fatalf("sql.Open() error = %v", err)
|
||||
}
|
||||
if _, err := raw.ExecContext(t.Context(), `CREATE TABLE turns (
|
||||
turn_id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 0,
|
||||
owner_id TEXT NOT NULL DEFAULT '',
|
||||
input_message_id TEXT NOT NULL DEFAULT '',
|
||||
assistant_message_id TEXT NOT NULL DEFAULT '',
|
||||
partial_text TEXT NOT NULL DEFAULT '',
|
||||
response_text TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
t.Fatalf("create legacy turns table: %v", err)
|
||||
}
|
||||
if _, err := raw.ExecContext(t.Context(), `INSERT INTO turns (
|
||||
turn_id, session_id, source_id, status, attempt, owner_id,
|
||||
input_message_id, assistant_message_id, partial_text, response_text,
|
||||
error, created_at, updated_at
|
||||
) VALUES ('legacy-turn', 'legacy-session', 'legacy-source', 'completed', 1, 'owner',
|
||||
'', 'assistant-1', '', 'legacy answer', '', 0, 0)`); err != nil {
|
||||
t.Fatalf("insert legacy turn row: %v", err)
|
||||
}
|
||||
if err := raw.Close(); err != nil {
|
||||
t.Fatalf("close legacy handle: %v", err)
|
||||
}
|
||||
|
||||
store, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore() on legacy schema error = %v", err)
|
||||
}
|
||||
defer func() { _ = store.Close() }()
|
||||
ledger, ok := store.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want TurnLedger", store)
|
||||
}
|
||||
turn, ok, err := ledger.GetTurn(t.Context(), "legacy-turn")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() after migration = %#v, %v, %v; want the legacy row", turn, ok, err)
|
||||
}
|
||||
if turn.ResponseText != "legacy answer" || len(turn.ToolCalls) != 0 {
|
||||
t.Fatalf("migrated turn = %#v, want legacy text with no tool calls", turn)
|
||||
}
|
||||
|
||||
// Reopening a second time must not fail on the "duplicate column name"
|
||||
// the migration produces once the column already exists.
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
reopened, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("second OpenSQLiteSessionStore() error = %v", err)
|
||||
}
|
||||
_ = reopened.Close()
|
||||
}
|
||||
|
||||
func TestSQLiteSessionStoreRejectsExcessiveSearchQueries(t *testing.T) {
|
||||
st := newTestSQLiteStore(t)
|
||||
tests := []struct {
|
||||
|
||||
@@ -147,6 +147,33 @@ func (f DeltaFunc) Delta(text string) {
|
||||
// ToolCall discards the event: a text-only caller has nowhere to put it.
|
||||
func (f DeltaFunc) ToolCall(ToolCallEvent) {}
|
||||
|
||||
// toolCallRecorder wraps a TurnStream, forwarding every event to it
|
||||
// unchanged while also recording the tool calls in arrival order. The
|
||||
// recording happens regardless of whether next is nil (a non-streaming
|
||||
// caller, e.g. Router.LLM) so a turn's tool activity is captured for
|
||||
// persistence and later replay even when nothing rendered it live the first
|
||||
// time.
|
||||
type toolCallRecorder struct {
|
||||
next TurnStream
|
||||
events []ToolCallEvent
|
||||
}
|
||||
|
||||
// Delta forwards the fragment to the wrapped stream, if any.
|
||||
func (r *toolCallRecorder) Delta(text string) {
|
||||
if r.next != nil {
|
||||
r.next.Delta(text)
|
||||
}
|
||||
}
|
||||
|
||||
// ToolCall records the event, then forwards it to the wrapped stream, if
|
||||
// any.
|
||||
func (r *toolCallRecorder) ToolCall(event ToolCallEvent) {
|
||||
r.events = append(r.events, event)
|
||||
if r.next != nil {
|
||||
r.next.ToolCall(event)
|
||||
}
|
||||
}
|
||||
|
||||
// firstNonEmptyLine returns the first line of s that holds something other
|
||||
// than whitespace, trimmed. It returns "" when there is no such line.
|
||||
func firstNonEmptyLine(s string) string {
|
||||
|
||||
@@ -125,8 +125,20 @@ func (r *TurnRunner) RespondStream(ctx context.Context, msg Message, stream Turn
|
||||
|
||||
// Run executes one turn, reporting progress to stream. A completed duplicate
|
||||
// source message replays its durable response without invoking the model
|
||||
// again; the replay reaches stream as one whole-text fragment, since there is
|
||||
// no generation left to watch.
|
||||
// again; the replay reaches stream as the turn's recorded tool-call events,
|
||||
// in the order they originally ran, followed by one whole-text fragment for
|
||||
// the answer, since there is no generation left to watch. Replaying the tool
|
||||
// events first is what keeps a redelivered or restart-recovered duplicate
|
||||
// from disagreeing with the original about what ran.
|
||||
//
|
||||
// The prior-reply fallback below (matching an already-answered source
|
||||
// message by conversation history rather than by turn ID) is deliberately
|
||||
// text-only: it exists for the narrow race where a process crashed after
|
||||
// saving the assistant message but before the turn record was marked
|
||||
// completed, so the original turn's structured tool record -- saved in the
|
||||
// same call as ResponseText -- never made it to durable storage either. It
|
||||
// has nothing to replay, so it must not synthesize a tool block it cannot
|
||||
// verify.
|
||||
//
|
||||
// stream may be nil, which means the caller renders only the returned reply.
|
||||
func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (string, error) {
|
||||
@@ -172,8 +184,13 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
}
|
||||
switch claim {
|
||||
case TurnClaimCompleted:
|
||||
if stream != nil && turn.ResponseText != "" {
|
||||
stream.Delta(turn.ResponseText)
|
||||
if stream != nil {
|
||||
for _, event := range turn.ToolCalls {
|
||||
stream.ToolCall(event)
|
||||
}
|
||||
if turn.ResponseText != "" {
|
||||
stream.Delta(turn.ResponseText)
|
||||
}
|
||||
}
|
||||
return turn.ResponseText, nil
|
||||
case TurnClaimInProgress:
|
||||
@@ -284,10 +301,11 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
view.Messages = append(
|
||||
[]CompressedMessage{{Role: "system", Content: systemPrompt}}, view.Messages...,
|
||||
)
|
||||
recorder := &toolCallRecorder{next: stream}
|
||||
text, err := prepared.Generate(ctx, TurnModelRequest{
|
||||
Messages: view.Messages,
|
||||
MaxTokens: modelDetails.MaxOutputTokens,
|
||||
}, stream)
|
||||
}, recorder)
|
||||
if err != nil {
|
||||
return "", r.failTurn(ctx, turn, err)
|
||||
}
|
||||
@@ -298,6 +316,7 @@ func (r *TurnRunner) Run(ctx context.Context, msg Message, stream TurnStream) (s
|
||||
turn.Status = TurnStatusCompleted
|
||||
turn.AssistantMessageID = assistant.MessageID
|
||||
turn.ResponseText = text
|
||||
turn.ToolCalls = recorder.events
|
||||
turn.Error = ""
|
||||
turn.UpdatedAt = time.Now().UTC()
|
||||
if err := r.saveTurn(ctx, turn); err != nil {
|
||||
|
||||
@@ -39,7 +39,11 @@ var ErrTurnOwnershipLost = errors.New("chat turn ownership lost")
|
||||
|
||||
// TurnRecord is the durable identity and outcome projection for one chat
|
||||
// generation. ResponseText is retained so a completed duplicate can replay
|
||||
// without depending on the bounded model-context history window.
|
||||
// without depending on the bounded model-context history window. ToolCalls
|
||||
// is retained for the same reason and saved in the same call as
|
||||
// ResponseText, so a completed-duplicate replay (NATS redelivery, restart
|
||||
// RecoverTurns redelivery) can narrate the tool activity that produced the
|
||||
// answer instead of showing a tool-less duplicate beside the original.
|
||||
type TurnRecord struct {
|
||||
TurnID string
|
||||
SessionID string
|
||||
@@ -51,6 +55,7 @@ type TurnRecord struct {
|
||||
AssistantMessageID string
|
||||
PartialText string
|
||||
ResponseText string
|
||||
ToolCalls []ToolCallEvent
|
||||
Error string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
|
||||
@@ -207,6 +207,68 @@ func TestSessionDeleteRemovesTurns(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTurnLedgerPersistsToolCallsAcrossClaimAndReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
ctx := context.Background()
|
||||
store, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore() error = %v", err)
|
||||
}
|
||||
ledger, ok := store.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want TurnLedger", store)
|
||||
}
|
||||
turn := TurnRecord{
|
||||
TurnID: CanonicalTurnID("session-tools", "source-tools"),
|
||||
SessionID: "session-tools",
|
||||
SourceID: "source-tools",
|
||||
OwnerID: "process-a",
|
||||
Status: TurnStatusAccepted,
|
||||
}
|
||||
claimed, claim, err := ledger.ClaimTurn(ctx, turn)
|
||||
if err != nil || claim != TurnClaimOwned {
|
||||
t.Fatalf("ClaimTurn() = %#v, %q, %v", claimed, claim, err)
|
||||
}
|
||||
claimed.Status = TurnStatusCompleted
|
||||
claimed.ResponseText = "answer"
|
||||
claimed.AssistantMessageID = "assistant-1"
|
||||
claimed.ToolCalls = []ToolCallEvent{
|
||||
{ID: "call-1", Name: "shell", Parameters: `{"cmd":"ls"}`, Output: "a.txt"},
|
||||
{ID: "call-2", Name: "grep", Err: "no matches"},
|
||||
}
|
||||
if err := ledger.SaveTurn(ctx, claimed); err != nil {
|
||||
t.Fatalf("SaveTurn() error = %v", err)
|
||||
}
|
||||
|
||||
completed, claim, err := ledger.ClaimTurn(ctx, turn)
|
||||
if err != nil || claim != TurnClaimCompleted {
|
||||
t.Fatalf("re-claim = %#v, %q, %v", completed, claim, err)
|
||||
}
|
||||
if len(completed.ToolCalls) != 2 || completed.ToolCalls[0].Name != "shell" || completed.ToolCalls[1].Err != "no matches" {
|
||||
t.Fatalf("re-claimed tool calls = %#v, want the two saved events in order", completed.ToolCalls)
|
||||
}
|
||||
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
reopened, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen error = %v", err)
|
||||
}
|
||||
defer func() { _ = reopened.Close() }()
|
||||
reopenedLedger, ok := reopened.(TurnLedger)
|
||||
if !ok {
|
||||
t.Fatalf("reopened store is %T, want TurnLedger", reopened)
|
||||
}
|
||||
got, ok, err := reopenedLedger.GetTurn(ctx, turn.TurnID)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() after reopen = %#v, %v, %v", got, ok, err)
|
||||
}
|
||||
if len(got.ToolCalls) != 2 || got.ToolCalls[0].Name != "shell" || got.ToolCalls[1].Name != "grep" {
|
||||
t.Fatalf("reopened tool calls = %#v, want the two saved events in order", got.ToolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteTurnLedgerSurvivesReopen(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -99,6 +99,85 @@ func TestTurnRunnerReplayStreamsStoredReplyOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A completed-duplicate replay (NATS redelivery, restart RecoverTurns
|
||||
// redelivery) must narrate the same tool activity the original turn did, in
|
||||
// the same order, ahead of the answer -- otherwise the duplicate disagrees
|
||||
// with the original about what ran (archie-core-ippu).
|
||||
func TestTurnRunnerReplayCompletedDuplicateReplaysToolCalls(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "done"}
|
||||
prepared.generateStream = func(stream TurnStream) (string, error) {
|
||||
stream.ToolCall(ToolCallEvent{Name: "shell", Output: "exit 0"})
|
||||
stream.Delta("done")
|
||||
return "done", nil
|
||||
}
|
||||
runner := newStreamTestRunner(t, prepared)
|
||||
msg := Message{From: "user", ChannelID: "chat-1", SourceID: "source-1", Text: "hello"}
|
||||
|
||||
if _, err := runner.Run(context.Background(), msg, &recordingStream{}); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
|
||||
sink := &recordingStream{}
|
||||
reply, err := runner.Run(context.Background(), msg, sink)
|
||||
if err != nil {
|
||||
t.Fatalf("replay Run() error = %v", err)
|
||||
}
|
||||
if reply != "done" {
|
||||
t.Fatalf("replayed reply = %q", reply)
|
||||
}
|
||||
want := []string{"tool:shell:exit 0", "text:done"}
|
||||
if len(sink.events) != len(want) {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
for i := range want {
|
||||
if sink.events[i] != want[i] {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
}
|
||||
if prepared.generateN != 1 {
|
||||
t.Fatalf("Generate calls = %d, want 1 -- a replay must not call the model", prepared.generateN)
|
||||
}
|
||||
|
||||
turn, ok, err := runner.Ledger.GetTurn(context.Background(), CanonicalTurnID("chat-1", "source-1"))
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("GetTurn() = %#v, %v, %v; want persisted turn", turn, ok, err)
|
||||
}
|
||||
if len(turn.ToolCalls) != 1 || turn.ToolCalls[0].Name != "shell" {
|
||||
t.Fatalf("persisted turn tool calls = %#v, want one recorded shell call", turn.ToolCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// A turn's recorded tool activity is captured even when the original caller
|
||||
// did not stream (Router.LLM, a nil sink): a later duplicate delivered to a
|
||||
// streaming caller must still be able to replay it.
|
||||
func TestTurnRunnerRecordsToolCallsEvenWithoutOriginalStream(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "done"}
|
||||
prepared.generateStream = func(stream TurnStream) (string, error) {
|
||||
stream.ToolCall(ToolCallEvent{Name: "grep", Output: "3 matches"})
|
||||
return "done", nil
|
||||
}
|
||||
runner := newStreamTestRunner(t, prepared)
|
||||
msg := Message{From: "user", ChannelID: "chat-1", SourceID: "source-1", Text: "hello"}
|
||||
|
||||
if _, err := runner.Run(context.Background(), msg, nil); err != nil {
|
||||
t.Fatalf("first Run() error = %v", err)
|
||||
}
|
||||
|
||||
sink := &recordingStream{}
|
||||
if _, err := runner.Run(context.Background(), msg, sink); err != nil {
|
||||
t.Fatalf("replay Run() error = %v", err)
|
||||
}
|
||||
want := []string{"tool:grep:3 matches", "text:done"}
|
||||
if len(sink.events) != len(want) {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
for i := range want {
|
||||
if sink.events[i] != want[i] {
|
||||
t.Fatalf("replay stream events = %v, want %v", sink.events, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A nil sink means "not streaming": the turn still runs and returns its reply.
|
||||
func TestTurnRunnerAcceptsNoStream(t *testing.T) {
|
||||
prepared := &turnTestPreparedModel{reply: "quiet answer"}
|
||||
|
||||
Reference in new issue
Block a user