fix(chat): keep long turns alive, record and log why a turn failed (#1813)
deploy / plan (push) Failing after 1s
deploy / gate (push) Skipped
deploy / archied-image (push) Skipped
deploy / agent-image (push) Skipped
deploy / dist-zip (push) Skipped

This commit is contained in:
sam committed 2026-11-08 23:03:04 +11:00
1 parent 10d23e3e06
commit 074ebbd931
3 files changed
+52 -1

No files matched your search

+13
View File
@@ -518,6 +518,10 @@ func (r *TurnRunner) generateAndComplete(
MaxOutputTokens: prep.modelDetails.MaxOutputTokens,
}, recorder)
if err != nil {
turn.ToolCalls = recorder.events
if n := len(recorder.events); n > 0 {
err = fmt.Errorf("%w (last completed step: %s)", err, recorder.events[n-1].Name)
}
return "", r.failTurn(ctx, turn, err)
}
assistant := messaging.Message{
@@ -578,6 +582,15 @@ func (r *TurnRunner) failTurn(ctx context.Context, turn TurnRecord, cause error)
}
turn.Error = cause.Error()
turn.UpdatedAt = time.Now().UTC()
if r.Log != nil {
level := slog.LevelError
if turn.Status == TurnStatusCancelled {
level = slog.LevelWarn
}
r.Log.Log(ctx, level, "chat turn did not complete",
"turn", turn.TurnID, "session", turn.SessionID, "status", string(turn.Status),
"tool_calls", len(turn.ToolCalls), "error", cause)
}
if err := r.saveTurn(ctx, turn); err != nil {
return errors.Join(cause, fmt.Errorf("save failed chat turn: %w", err))
}
+26
View File
@@ -417,6 +417,8 @@ func (a *chatAPI) chatShowToolCalls(ctx context.Context) bool {
return view.Chat.ShowToolCalls
}
const chatKeepaliveInterval = 15 * time.Second
func (a *chatAPI) handleChatStream(w http.ResponseWriter, r *http.Request) {
chat, ok := a.chatReady(w)
if !ok {
@@ -445,6 +447,30 @@ func (a *chatAPI) handleChatStream(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "data: %s\n\n", payload)
flusher.Flush()
}
// A tool can run for minutes without a frame; a comment line keeps the
// browser's idle timer and any proxy from cutting the stream.
stopKeepalive := make(chan struct{})
keepaliveDone := make(chan struct{})
defer func() {
close(stopKeepalive)
<-keepaliveDone
}()
go func() {
defer close(keepaliveDone)
tick := time.NewTicker(chatKeepaliveInterval)
defer tick.Stop()
for {
select {
case <-stopKeepalive:
return
case <-tick.C:
writeMu.Lock()
_, _ = fmt.Fprint(w, ": keepalive\n\n")
flusher.Flush()
writeMu.Unlock()
}
}
}()
asker := dashboardAsker{asks: &a.asks, write: func(frame chatStreamEvent) { writeChatEvent(frame, "") }}
events, err := chat.Contract.Stream(withDashboardAsks(r.Context(), asker), msg)
if err != nil {
+13 -1
View File
@@ -626,6 +626,9 @@ export async function sendMessage(retryOpts?: RetryOptions): Promise<void> {
await reader.cancel().catch(() => {});
return;
}
// Any bytes, including the server's keepalive, show the turn is alive;
// a question waiting on the operator stops the clock until answered.
if (!done && !streamingTurn.value?.ask) arm();
buffer += decoder.decode(value || new Uint8Array(), { stream: !done });
const frames = buffer.split("\n\n");
buffer = frames.pop() || "";
@@ -713,7 +716,11 @@ export async function sendMessage(retryOpts?: RetryOptions): Promise<void> {
statusText.value = "Stopped";
} else {
streamingTurn.value = {
text: `Unable to complete that turn: ${failed.message || failed}`,
text: `Unable to complete that turn: ${
timedOut
? `no activity for ${STREAM_TIMEOUT_MS / 60000} minutes${lastStep(activeTools)}`
: failed.message || failed
}`,
tools: activeTools,
isError: true,
turn: turn ? retryChatTurn(turn) : undefined,
@@ -736,6 +743,11 @@ export async function sendMessage(retryOpts?: RetryOptions): Promise<void> {
}
}
function lastStep(tools: ChatToolCall[]): string {
const last = tools.filter((tool) => tool.name || tool.tool).at(-1);
return last ? `, last step: ${last.name || last.tool}` : "";
}
export function stopTurn(): void {
statusText.value = "Stopping…";
if (currentSession.value) {