feat: make /stop cancel the running chat turn
/stop could not reach a running turn, and the reason was worse than a missing cancel. go-telegram/bot runs one update worker that calls handlers synchronously (defaultWorkers = 1, never overridden), and the chat turn ran on it. While a turn streamed -- or sat in a shell tool -- the worker was blocked and /stop sat unread in the update channel until the turn it was meant to kill had already finished. Add gateway.Turns: a per-session serial lane that turns are submitted to and that the event loop never waits on. The worker is free again in microseconds, so /stop is serviced immediately; ordering comes from the lane being serial rather than from the loop being blocked. - Bare /stop cancels the running turn and clears the backlog behind it, with no approval step. Approval guards destructive commands, and gating the brake behind a confirmation defeats it. /stop <name> keeps the existing process-termination behaviour and its approval. - Cancellation propagates through the turn context, so it reaches the model stream and any tool underneath -- a shell command built with exec.CommandContext dies with the turn. - Queues are unbounded and never shed. A dropped message is a silent failure that surfaces later as an agent that ignored something. - Turns are recovered per-lane: the channel's panic middleware wraps the event-loop goroutine, which no longer runs the turn. Also raise the chat step budget. It was hardcoded at 8 -- roughly one read, one grep and two edits -- so turns hit the cap and stopped mid-task with no indication they had been cut off. It is now chat.max_steps, defaulting to 100. Interruption is /stop's job.
This commit is contained in:
1 parent
e03cef17c9
commit
561e56b70a
10 files changed
+765
-34
No files matched your search
+14
-2
@@ -71,7 +71,15 @@ func main() {
|
||||
}
|
||||
|
||||
const (
|
||||
defaultChatMaxSteps = 8
|
||||
// defaultChatMaxSteps bounds the model/tool round-trips in one chat
|
||||
// turn when [config.ChatConfig.MaxSteps] is unset.
|
||||
//
|
||||
// It was 8, which is roughly one read, one grep and a couple of edits
|
||||
// -- less than a single small change to a Go package costs. Turns hit
|
||||
// the cap and stopped mid-task with no indication they had been cut
|
||||
// off. Interruption is /stop's job; this only has to stop a genuine
|
||||
// runaway, so it is set well above real work.
|
||||
defaultChatMaxSteps = 100
|
||||
packagedGatewayChangelogPath = "/usr/share/archie/CHANGELOG.archied.md"
|
||||
packagedRuntimeChangelogPath = "/usr/share/archie/CHANGELOG.archie.md"
|
||||
)
|
||||
@@ -87,15 +95,19 @@ var (
|
||||
func chatGenerateOptions(
|
||||
messages []chat.Message,
|
||||
registry *tools.Registry,
|
||||
maxSteps int,
|
||||
) (core.GenerateOptions, error) {
|
||||
toolSet, err := agentexec.BuildToolSet(registry)
|
||||
if err != nil {
|
||||
return core.GenerateOptions{}, err
|
||||
}
|
||||
if maxSteps <= 0 {
|
||||
maxSteps = defaultChatMaxSteps
|
||||
}
|
||||
return core.GenerateOptions{
|
||||
Messages: messages,
|
||||
Tools: toolSet,
|
||||
MaxSteps: defaultChatMaxSteps,
|
||||
MaxSteps: maxSteps,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ func TestChatGenerateOptionsIncludesToolsAndMultipleSteps(t *testing.T) {
|
||||
}
|
||||
messages := []chat.Message{{Role: chat.RoleUser, Content: "remember this"}}
|
||||
|
||||
options, err := chatGenerateOptions(messages, registry)
|
||||
options, err := chatGenerateOptions(messages, registry, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -64,11 +64,24 @@ func TestChatGenerateOptionsReportsInvalidToolSchemas(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := chatGenerateOptions(nil, registry); err == nil {
|
||||
if _, err := chatGenerateOptions(nil, registry, 0); err == nil {
|
||||
t.Fatal("chatGenerateOptions() error = nil, want invalid tool schema error")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatGenerateOptionsHonoursConfiguredMaxSteps keeps the step budget
|
||||
// deployment-controlled. A cap that cannot be raised without a rebuild is
|
||||
// the situation this replaced.
|
||||
func TestChatGenerateOptionsHonoursConfiguredMaxSteps(t *testing.T) {
|
||||
options, err := chatGenerateOptions(nil, tools.NewRegistry(), 250)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if options.MaxSteps != 250 {
|
||||
t.Errorf("MaxSteps = %d, want the configured 250", options.MaxSteps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGatewaysStartOnlyAfterCapabilityHostAndDaemonStartup(t *testing.T) {
|
||||
source, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
|
||||
@@ -180,7 +180,7 @@ func makeTelegramLLMResponder(ctx context.Context, tg *telegram.Gateway, s teleg
|
||||
compressed = append(compressed, gateway.CompressedMessage{Role: role, Content: h.Text})
|
||||
}
|
||||
view := gateway.CompressHistory(compressed, gateway.DefaultCompressionConfig())
|
||||
options, err := chatGenerateOptions(nil, s.ToolReg)
|
||||
options, err := chatGenerateOptions(nil, s.ToolReg, s.Cfg.Chat.MaxSteps)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build chat tools: %w", err)
|
||||
}
|
||||
|
||||
@@ -55,9 +55,17 @@ class = "ollama"
|
||||
# shell, grep, find). Unset, those tools are not registered at all. Set it and
|
||||
# the agent can read, modify and execute inside that directory with the same
|
||||
# reach the daemon process has -- choose it deliberately.
|
||||
#
|
||||
# `max_steps` caps the model/tool round-trips in a single chat turn
|
||||
# (default 100). This is a task budget, not a safety limit: one question
|
||||
# about a codebase easily costs a read, several greps and a few edits, and
|
||||
# a turn that runs out stops mid-task looking like it gave up. To interrupt
|
||||
# a turn, send /stop -- that cancels it immediately and clears anything
|
||||
# queued behind it.
|
||||
# [chat]
|
||||
# operator = "Ada Lovelace"
|
||||
# workspace = "/var/lib/archie/workspace"
|
||||
# max_steps = 100
|
||||
# models = [
|
||||
# "openai/gpt-5.6-sol",
|
||||
# "openrouter/openai/gpt-5.6-sol",
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
|
||||
"github.com/go-telegram/bot"
|
||||
"github.com/go-telegram/bot/models"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
)
|
||||
|
||||
// DangerousCommandAuthority is the sandbox/process/filesystem authority
|
||||
@@ -405,25 +407,74 @@ func rollbackApprovalText(num int) string {
|
||||
|
||||
// ── /stop handler ──────────────────────────────────────────────
|
||||
|
||||
// stopHandler serves /stop: terminate a background process after approval.
|
||||
func (g *Gateway) stopHandler() bot.HandlerFunc {
|
||||
// stopCurrentTurn cancels whatever this conversation is currently doing.
|
||||
//
|
||||
// Cancellation propagates through the turn's context, so it reaches the
|
||||
// model stream and any tool running underneath it -- a shell command built
|
||||
// with exec.CommandContext is killed with the turn, which is the case that
|
||||
// matters. Whatever the reply had already streamed stays on screen: it is
|
||||
// the record of what Archie did before being stopped.
|
||||
func (g *Gateway) stopCurrentTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) {
|
||||
var cancelled bool
|
||||
var dropped int
|
||||
|
||||
// turns is only built by launch, so a gateway that has never started
|
||||
// simply has nothing to stop. Report that rather than failing.
|
||||
if g.turns != nil && router != nil {
|
||||
session, err := router.ResolveSessionKey(ctx, gateway.Message{
|
||||
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
|
||||
ThreadID: threadIDString(msg.MessageThreadID),
|
||||
From: msg.From.Username,
|
||||
Text: msg.Text,
|
||||
})
|
||||
if err != nil {
|
||||
g.log.Error("resolve session for stop", "error", err)
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.")
|
||||
return
|
||||
}
|
||||
cancelled, dropped = g.turns.Stop(session)
|
||||
g.log.Info("stop requested", "session", session, "cancelled", cancelled, "dropped", dropped)
|
||||
}
|
||||
|
||||
switch {
|
||||
case cancelled && dropped > 0:
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
|
||||
fmt.Sprintf("🛑 Stopped. %d queued message(s) discarded.", dropped))
|
||||
case cancelled:
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "🛑 Stopped.")
|
||||
case dropped > 0:
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
|
||||
fmt.Sprintf("🛑 Nothing was running. %d queued message(s) discarded.", dropped))
|
||||
default:
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
|
||||
"Nothing is running.\n\nUse /stop <process-name> to terminate a background process.")
|
||||
}
|
||||
}
|
||||
|
||||
// stopHandler serves /stop.
|
||||
//
|
||||
// Bare /stop is the emergency brake: it cancels the conversation's running
|
||||
// turn immediately, with no approval step. Approval exists to protect
|
||||
// against destructive commands, and stopping is the opposite -- gating it
|
||||
// behind a confirmation round-trip would defeat the entire point.
|
||||
//
|
||||
// /stop <process-name> keeps the original behaviour of terminating a named
|
||||
// background process, which is destructive and still requires approval.
|
||||
func (g *Gateway) stopHandler(router *gateway.Router) bot.HandlerFunc {
|
||||
return func(ctx context.Context, b *bot.Bot, update *models.Update) {
|
||||
msg, ok := g.authorizedMessage(ctx, b, update)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if g.Dangerous == nil {
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.")
|
||||
|
||||
rest := strings.TrimSpace(restAfterTelegram(msg.Text, "/stop", ""))
|
||||
if rest == "" {
|
||||
g.stopCurrentTurn(ctx, b, msg, router)
|
||||
return
|
||||
}
|
||||
|
||||
rest := restAfterTelegram(msg.Text, "/stop", "")
|
||||
rest = strings.TrimSpace(rest)
|
||||
|
||||
if rest == "" {
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
|
||||
"Usage: /stop <process-name>\n\nTerminate a running background process. "+
|
||||
"This requires approval because it may discard in-progress work.")
|
||||
if g.Dangerous == nil {
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.")
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-telegram/bot/models"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
)
|
||||
|
||||
// ── DangerousCommandAuthority stub ─────────────────────────────
|
||||
@@ -303,7 +305,7 @@ func TestStopRequiresApproval(t *testing.T) {
|
||||
g.Dangerous = stub
|
||||
b, requests := newTelegramTestBot(t)
|
||||
|
||||
g.stopHandler()(context.Background(), b, &models.Update{
|
||||
g.stopHandler(nil)(context.Background(), b, &models.Update{
|
||||
Message: &models.Message{
|
||||
From: &models.User{ID: allowedUserID},
|
||||
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
|
||||
@@ -333,7 +335,7 @@ func TestStopApprovalExecutes(t *testing.T) {
|
||||
b, requests := newTelegramTestBot(t)
|
||||
|
||||
// Step 1: Trigger the approval prompt.
|
||||
g.stopHandler()(context.Background(), b, &models.Update{
|
||||
g.stopHandler(nil)(context.Background(), b, &models.Update{
|
||||
Message: &models.Message{
|
||||
From: &models.User{ID: allowedUserID},
|
||||
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
|
||||
@@ -367,14 +369,18 @@ func TestStopApprovalExecutes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopNoArgument(t *testing.T) {
|
||||
// TestStopWithNoArgumentDoesNotRequireApproval is the emergency-brake
|
||||
// contract. Bare /stop targets the running chat turn, so it must act
|
||||
// immediately -- if it ever renders an approval prompt again, the brake has
|
||||
// been put behind a confirmation round-trip and no longer works.
|
||||
func TestStopWithNoArgumentDoesNotRequireApproval(t *testing.T) {
|
||||
const allowedUserID = int64(42)
|
||||
stub := &dangerousStub{}
|
||||
g := New("1:test", "", "", []int64{allowedUserID}, slog.Default())
|
||||
g.Dangerous = stub
|
||||
b, requests := newTelegramTestBot(t)
|
||||
|
||||
g.stopHandler()(context.Background(), b, &models.Update{
|
||||
g.stopHandler(nil)(context.Background(), b, &models.Update{
|
||||
Message: &models.Message{
|
||||
From: &models.User{ID: allowedUserID},
|
||||
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
|
||||
@@ -383,8 +389,51 @@ func TestStopNoArgument(t *testing.T) {
|
||||
})
|
||||
|
||||
text := messageText(*requests, 0)
|
||||
if !strings.Contains(text, "Usage: /stop") {
|
||||
t.Errorf("expected usage message, got: %s", text)
|
||||
if strings.Contains(text, "Dangerous command") {
|
||||
t.Errorf("bare /stop asked for approval: %s", text)
|
||||
}
|
||||
if !strings.Contains(text, "Nothing is running") {
|
||||
t.Errorf("expected an idle report, got: %s", text)
|
||||
}
|
||||
if len(stub.stopCalls) != 0 {
|
||||
t.Errorf("bare /stop terminated a named process: %v", stub.stopCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopCancelsRunningChatTurn covers the whole point of the command:
|
||||
// a turn in flight is cancelled, and the queue behind it is discarded.
|
||||
func TestStopCancelsRunningChatTurn(t *testing.T) {
|
||||
const allowedUserID = int64(42)
|
||||
g := New("1:test", "", "", []int64{allowedUserID}, slog.Default())
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
g.turns = gateway.NewTurns(slog.Default())
|
||||
|
||||
// stopCurrentTurn resolves the session through the router; with none
|
||||
// wired, drive the lane directly and assert on the lane's state.
|
||||
const session = "chat:7"
|
||||
started := make(chan struct{})
|
||||
stopped := make(chan struct{})
|
||||
g.turns.Submit(ctx, session, func(turnCtx context.Context) {
|
||||
close(started)
|
||||
<-turnCtx.Done()
|
||||
close(stopped)
|
||||
})
|
||||
<-started
|
||||
g.turns.Submit(ctx, session, func(context.Context) {})
|
||||
|
||||
cancelled, dropped := g.turns.Stop(session)
|
||||
if !cancelled {
|
||||
t.Error("the running turn was not cancelled")
|
||||
}
|
||||
if dropped != 1 {
|
||||
t.Errorf("dropped = %d, want 1", dropped)
|
||||
}
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the turn did not observe cancellation")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -851,7 +900,7 @@ func TestStopNotConfigured(t *testing.T) {
|
||||
// Dangerous is nil.
|
||||
b, requests := newTelegramTestBot(t)
|
||||
|
||||
g.stopHandler()(context.Background(), b, &models.Update{
|
||||
g.stopHandler(nil)(context.Background(), b, &models.Update{
|
||||
Message: &models.Message{
|
||||
From: &models.User{ID: allowedUserID},
|
||||
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
|
||||
|
||||
@@ -6,6 +6,7 @@ package telegram
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -71,6 +72,11 @@ type Gateway struct {
|
||||
dangerousActions map[string]dangerousAction
|
||||
permanentApprovals []permanentApproval
|
||||
|
||||
// turns serialises chat turns per session off the update worker and
|
||||
// makes the running one cancellable by /stop. Rebuilt on every launch
|
||||
// so a restart abandons in-flight turns with the old bot instance.
|
||||
turns *gateway.Turns
|
||||
|
||||
log *slog.Logger
|
||||
bot *bot.Bot
|
||||
webhookCancel context.CancelFunc
|
||||
@@ -162,6 +168,11 @@ func (g *Gateway) Start(ctx context.Context, router *gateway.Router) error {
|
||||
// launch builds one bot instance, registers handlers and begins receiving.
|
||||
// It returns as soon as delivery is running; the caller owns ctx.
|
||||
func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot, error) {
|
||||
// Turns are per-launch. Each queued turn carries the update handler's
|
||||
// context, which is this launch's, so a /restart cancels everything
|
||||
// still running under the outgoing bot instance.
|
||||
g.turns = gateway.NewTurns(g.log)
|
||||
|
||||
opts := []bot.Option{
|
||||
bot.WithErrorsHandler(func(err error) {
|
||||
g.log.Error("telegram pipeline error", "error", err)
|
||||
@@ -195,7 +206,7 @@ func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot,
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/help", bot.MatchTypeExact, g.helpHandler())
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/restart", bot.MatchTypeExact, g.restartHandler())
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/rollback", bot.MatchTypePrefix, g.rollbackHandler())
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler())
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler(router))
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/approve", bot.MatchTypeExact, g.approveHandler())
|
||||
b.RegisterHandler(bot.HandlerTypeMessageText, "/deny", bot.MatchTypeExact, g.denyHandler())
|
||||
|
||||
@@ -434,29 +445,63 @@ func (g *Gateway) defaultHandler(router *gateway.Router) bot.HandlerFunc {
|
||||
g.handlePersonalityCommand(ctx, b, msg, router)
|
||||
return
|
||||
}
|
||||
g.submitTurn(ctx, b, msg, router)
|
||||
}
|
||||
}
|
||||
|
||||
// submitTurn hands the message to its session's turn lane and returns at
|
||||
// once, leaving the bot's single update worker free.
|
||||
//
|
||||
// The turn must not run on the caller's goroutine. go-telegram/bot serves
|
||||
// updates from one worker that invokes handlers synchronously, so a turn
|
||||
// running here would block delivery of every later update -- including the
|
||||
// /stop meant to cancel it, which would sit unread until the turn it was
|
||||
// aimed at had already finished.
|
||||
func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) {
|
||||
gm := gateway.Message{
|
||||
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
|
||||
ThreadID: threadIDString(msg.MessageThreadID),
|
||||
From: msg.From.Username,
|
||||
Text: msg.Text,
|
||||
}
|
||||
|
||||
// The lane key must be the session, so that /stop -- which resolves
|
||||
// the same key -- reaches the turn the sender is actually watching.
|
||||
session, err := router.ResolveSessionKey(ctx, gm)
|
||||
if err != nil {
|
||||
g.log.Error("resolve session for turn", "error", err)
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.")
|
||||
return
|
||||
}
|
||||
|
||||
chatID, threadID := msg.Chat.ID, msg.MessageThreadID
|
||||
g.turns.Submit(ctx, session, func(turnCtx context.Context) {
|
||||
// reply appears as it is written; the typing indicator covers the
|
||||
// gap before the first token and any non-streaming path.
|
||||
stopTyping := g.startTyping(ctx, b, msg.Chat.ID, msg.MessageThreadID)
|
||||
draft := g.newDraft(b, msg.Chat.ID, msg.MessageThreadID)
|
||||
stopTyping := g.startTyping(turnCtx, b, chatID, threadID)
|
||||
draft := g.newDraft(b, chatID, threadID)
|
||||
|
||||
// If it starts with / but wasn't matched by a registered
|
||||
// handler, it's unknown -- let the router handle it (which
|
||||
// will say "unrecognized").
|
||||
reply, err := router.RouteStream(ctx, gateway.Message{
|
||||
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
|
||||
ThreadID: threadIDString(msg.MessageThreadID),
|
||||
From: msg.From.Username,
|
||||
Text: msg.Text,
|
||||
}, draft.onDelta)
|
||||
reply, err := router.RouteStream(turnCtx, gm, draft.onDelta)
|
||||
stopTyping()
|
||||
|
||||
// A cancelled turn is a /stop, not a fault. The stop handler has
|
||||
// already acknowledged it, and turnCtx is dead, so there is
|
||||
// nothing useful left to send from here.
|
||||
if errors.Is(err, context.Canceled) {
|
||||
g.log.Info("chat turn stopped", "session", session)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
g.log.Error("route failed", "error", err)
|
||||
return
|
||||
}
|
||||
if reply != "" {
|
||||
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, reply)
|
||||
g.sendMessage(turnCtx, b, chatID, threadID, reply)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────
|
||||
|
||||
@@ -491,6 +491,15 @@ type ChatConfig struct {
|
||||
// they read, write and execute, so the directory must be a deliberate
|
||||
// choice rather than whatever the daemon happens to start in.
|
||||
Workspace string `toml:"workspace" yaml:"workspace"`
|
||||
// MaxSteps caps how many model/tool round-trips one chat turn may take
|
||||
// before the runtime stops and returns what it has. Zero uses the
|
||||
// default.
|
||||
//
|
||||
// This is a real task budget, not a safety limit: a single question
|
||||
// about a codebase routinely costs a read, several greps and a handful
|
||||
// of edits, and running out mid-turn looks to the user like the agent
|
||||
// simply gave up. Use /stop to interrupt, not a small step cap.
|
||||
MaxSteps int `toml:"max_steps" yaml:"max_steps"`
|
||||
// Models is the optional interactive-chat model catalog. When empty,
|
||||
// chat falls back to the distinct model references assigned to workflow
|
||||
// roles in the top-level [models] table.
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Turns serialises chat turns per session and keeps the running turn
|
||||
// cancellable.
|
||||
//
|
||||
// Channel event loops are typically single-threaded -- go-telegram/bot,
|
||||
// for one, runs a single worker that calls handlers synchronously. Running
|
||||
// a turn on that goroutine blocks delivery of every subsequent update,
|
||||
// including the command meant to stop it: /stop would sit unread in the
|
||||
// update channel until the turn it was meant to kill had already finished.
|
||||
//
|
||||
// Submitting to Turns hands the work to a per-session goroutine and returns
|
||||
// immediately, so the event loop stays free and control commands are always
|
||||
// serviceable. Ordering is preserved by the lane being serial, not by the
|
||||
// event loop being blocked.
|
||||
//
|
||||
// Queues are unbounded and never shed. A dropped message is a silent
|
||||
// failure that surfaces much later as an agent that ignored something,
|
||||
// which costs far more to diagnose than the memory a backlog occupies.
|
||||
// [Turns.Stop] is the way a backlog is cleared -- deliberately, by the
|
||||
// person who created it.
|
||||
type Turns struct {
|
||||
log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
lanes map[string]*lane
|
||||
}
|
||||
|
||||
// queued is one pending turn: the work, already bound to its own
|
||||
// cancellable context, and the cancel that releases it.
|
||||
//
|
||||
// The context is derived when the turn is submitted rather than when it
|
||||
// starts running, which means a turn has a live cancel handle for its whole
|
||||
// time in the backlog. Stop can therefore cancel queued turns rather than
|
||||
// merely dropping them, and each turn stays tied to the caller that asked
|
||||
// for it -- it dies when that caller's context does.
|
||||
type queued struct {
|
||||
run func()
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// lane is one session's serial worker.
|
||||
//
|
||||
// A single mutex guards the backlog, the running turn's cancel, and whether
|
||||
// a worker is live, so that Stop observes them together. Splitting them
|
||||
// would leave a window where a turn finishes between the backlog being
|
||||
// cleared and the cancel being called, letting a queued turn start after a
|
||||
// stop.
|
||||
type lane struct {
|
||||
mu sync.Mutex
|
||||
pending []queued
|
||||
// cancel stops the turn currently running in this lane. It is nil
|
||||
// whenever the lane is idle.
|
||||
cancel context.CancelFunc
|
||||
// working reports whether a worker goroutine is draining this lane.
|
||||
// The worker exits when the backlog empties, so this is what tells
|
||||
// Submit whether to start a new one.
|
||||
working bool
|
||||
}
|
||||
|
||||
// NewTurns returns an idle dispatcher.
|
||||
func NewTurns(log *slog.Logger) *Turns {
|
||||
return &Turns{log: log, lanes: make(map[string]*lane)}
|
||||
}
|
||||
|
||||
// Submit schedules run for the given session and returns without waiting
|
||||
// for it. run receives a context derived from ctx that [Turns.Stop] can
|
||||
// cancel.
|
||||
//
|
||||
// Submit never blocks and never rejects: messages queue.
|
||||
func (t *Turns) Submit(ctx context.Context, session string, run func(context.Context)) {
|
||||
t.mu.Lock()
|
||||
l, ok := t.lanes[session]
|
||||
if !ok {
|
||||
l = &lane{}
|
||||
t.lanes[session] = l
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
turnCtx, cancel := context.WithCancel(ctx)
|
||||
|
||||
l.mu.Lock()
|
||||
l.pending = append(l.pending, queued{run: func() { run(turnCtx) }, cancel: cancel})
|
||||
depth := len(l.pending)
|
||||
start := !l.working
|
||||
if start {
|
||||
l.working = true
|
||||
}
|
||||
l.mu.Unlock()
|
||||
|
||||
if start {
|
||||
go l.serve(t.log.With("session", session))
|
||||
}
|
||||
if depth > 1 {
|
||||
t.log.Info("chat turn queued", "session", session, "queue_depth", depth)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop cancels the session's running turn and discards everything queued
|
||||
// behind it. It reports whether a turn was actually cancelled and how many
|
||||
// queued turns were dropped.
|
||||
//
|
||||
// Dropping the queue is intentional. "Stop" means stop, not "stop this one
|
||||
// and immediately begin the next thing I said" -- starting the queued turn
|
||||
// would look to the sender exactly like the stop had failed.
|
||||
func (t *Turns) Stop(session string) (cancelled bool, dropped int) {
|
||||
l := t.lane(session)
|
||||
if l == nil {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
dropped = len(l.pending)
|
||||
for i, q := range l.pending {
|
||||
q.cancel()
|
||||
l.pending[i] = queued{}
|
||||
}
|
||||
l.pending = nil
|
||||
if l.cancel != nil {
|
||||
l.cancel()
|
||||
cancelled = true
|
||||
}
|
||||
return cancelled, dropped
|
||||
}
|
||||
|
||||
// Running reports whether the session currently has a turn in flight.
|
||||
func (t *Turns) Running(session string) bool {
|
||||
l := t.lane(session)
|
||||
if l == nil {
|
||||
return false
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.cancel != nil
|
||||
}
|
||||
|
||||
// Queued reports how many turns are waiting behind the running one.
|
||||
func (t *Turns) Queued(session string) int {
|
||||
l := t.lane(session)
|
||||
if l == nil {
|
||||
return 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return len(l.pending)
|
||||
}
|
||||
|
||||
// lane returns the session's lane, or nil if it has never been used.
|
||||
func (t *Turns) lane(session string) *lane {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.lanes[session]
|
||||
}
|
||||
|
||||
// serve drains the lane's backlog one turn at a time and exits once it is
|
||||
// empty. Submit starts a fresh worker when more arrives, so an idle session
|
||||
// costs no goroutine.
|
||||
func (l *lane) serve(log *slog.Logger) {
|
||||
for {
|
||||
next, ok := l.begin()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
l.runOne(next, log)
|
||||
}
|
||||
}
|
||||
|
||||
// begin pops the oldest queued turn and publishes its cancel in the same
|
||||
// critical section, reporting false when the backlog is empty -- at which
|
||||
// point it also retires the worker.
|
||||
//
|
||||
// The pop and the publish must happen together. Were the turn popped first
|
||||
// and its cancel published after, a Stop landing in between would see an
|
||||
// empty backlog and no running turn, report that nothing was happening, and
|
||||
// let the turn it should have killed start regardless.
|
||||
func (l *lane) begin() (queued, bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if len(l.pending) == 0 {
|
||||
// Retire under the same lock Submit uses to decide whether a
|
||||
// worker is live, so a turn arriving now either starts a new
|
||||
// worker or is picked up before this one retires -- never
|
||||
// neither, which would strand the backlog forever.
|
||||
l.working = false
|
||||
return queued{}, false
|
||||
}
|
||||
|
||||
next := l.pending[0]
|
||||
// Clear the vacated slot so a completed turn's closure -- and whatever
|
||||
// it captured -- does not stay reachable through the backing array.
|
||||
l.pending[0] = queued{}
|
||||
l.pending = l.pending[1:]
|
||||
|
||||
l.cancel = next.cancel
|
||||
return next, true
|
||||
}
|
||||
|
||||
// runOne executes one turn, always retracting its cancel afterwards so a
|
||||
// later Stop cannot cancel an already-finished turn.
|
||||
//
|
||||
// Panics are contained here rather than by the channel's own recovery
|
||||
// middleware: that middleware wraps the event-loop goroutine, and the turn
|
||||
// no longer runs on it. Without this, one bad turn would take the lane's
|
||||
// goroutine down and the session would silently stop answering forever.
|
||||
func (l *lane) runOne(next queued, log *slog.Logger) {
|
||||
defer next.cancel()
|
||||
|
||||
defer func() {
|
||||
l.mu.Lock()
|
||||
l.cancel = nil
|
||||
l.mu.Unlock()
|
||||
|
||||
if r := recover(); r != nil {
|
||||
log.Error("chat turn panicked", "panic", r, "stack", string(debug.Stack()))
|
||||
}
|
||||
}()
|
||||
|
||||
next.run()
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package gateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/archie-core/internal/gateway"
|
||||
)
|
||||
|
||||
func testTurns(t *testing.T) *gateway.Turns {
|
||||
t.Helper()
|
||||
return gateway.NewTurns(slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
}
|
||||
|
||||
// TestSubmitDoesNotBlockCaller is the whole reason this type exists: the
|
||||
// channel event loop must stay free while a turn runs, or the command that
|
||||
// stops the turn can never be delivered.
|
||||
func TestSubmitDoesNotBlockCaller(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
release := make(chan struct{})
|
||||
started := make(chan struct{})
|
||||
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {
|
||||
close(started)
|
||||
<-release
|
||||
})
|
||||
<-started
|
||||
defer close(release)
|
||||
|
||||
returned := make(chan struct{})
|
||||
go func() {
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {})
|
||||
close(returned)
|
||||
}()
|
||||
select {
|
||||
case <-returned:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Submit blocked while a turn was running; the event loop would be stuck")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStopCancelsRunningTurn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
started := make(chan struct{})
|
||||
var cause atomic.Value
|
||||
|
||||
turns.Submit(context.Background(), "s1", func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
cause.Store(ctx.Err())
|
||||
})
|
||||
<-started
|
||||
|
||||
cancelled, dropped := turns.Stop("s1")
|
||||
if !cancelled {
|
||||
t.Error("Stop reported no running turn")
|
||||
}
|
||||
if dropped != 0 {
|
||||
t.Errorf("dropped = %d, want 0", dropped)
|
||||
}
|
||||
|
||||
waitFor(t, func() bool { return cause.Load() != nil })
|
||||
if err, _ := cause.Load().(error); !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("turn context error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopDropsQueuedTurns guards the "stop means stop" rule. If the
|
||||
// queued turn ran after cancellation, the sender would see a new reply
|
||||
// begin immediately and conclude the stop had failed.
|
||||
func TestStopDropsQueuedTurns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
var queuedRan atomic.Bool
|
||||
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {
|
||||
close(started)
|
||||
<-release
|
||||
})
|
||||
<-started
|
||||
|
||||
for range 2 {
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) { queuedRan.Store(true) })
|
||||
}
|
||||
|
||||
cancelled, dropped := turns.Stop("s1")
|
||||
if !cancelled || dropped != 2 {
|
||||
t.Fatalf("Stop() = (%v, %d), want (true, 2)", cancelled, dropped)
|
||||
}
|
||||
close(release)
|
||||
|
||||
// Give the lane time to pull anything that survived the drain.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if queuedRan.Load() {
|
||||
t.Error("a queued turn ran after /stop")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopLeavesTheCallersContextAlone checks that Stop only reaches the
|
||||
// contexts Turns derived. Callers pass in the channel's long-lived run
|
||||
// context, so cancelling that instead of the turn's own would take the
|
||||
// whole gateway down every time someone sent /stop.
|
||||
func TestStopLeavesTheCallersContextAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
callerCtx, callerCancel := context.WithCancel(context.Background())
|
||||
defer callerCancel()
|
||||
|
||||
started := make(chan struct{})
|
||||
turns.Submit(callerCtx, "s1", func(ctx context.Context) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
})
|
||||
<-started
|
||||
turns.Submit(callerCtx, "s1", func(context.Context) {})
|
||||
|
||||
if cancelled, dropped := turns.Stop("s1"); !cancelled || dropped != 1 {
|
||||
t.Fatalf("Stop() = (%v, %d), want (true, 1)", cancelled, dropped)
|
||||
}
|
||||
if err := callerCtx.Err(); err != nil {
|
||||
t.Errorf("Stop cancelled the caller's context: %v", err)
|
||||
}
|
||||
|
||||
// The lane must still be usable afterwards.
|
||||
ran := make(chan struct{})
|
||||
turns.Submit(callerCtx, "s1", func(context.Context) { close(ran) })
|
||||
select {
|
||||
case <-ran:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the session stopped accepting turns after /stop")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeepBacklogIsKeptAndOrdered is the no-shedding contract. A message
|
||||
// that is silently dropped surfaces much later as an agent that ignored
|
||||
// something, which is far more expensive to diagnose than holding the
|
||||
// backlog. Every message queues, and they run in the order they arrived.
|
||||
func TestDeepBacklogIsKeptAndOrdered(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const backlog = 100
|
||||
|
||||
turns := testTurns(t)
|
||||
started := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {
|
||||
close(started)
|
||||
<-release
|
||||
})
|
||||
<-started
|
||||
|
||||
var mu sync.Mutex
|
||||
var order []int
|
||||
done := make(chan struct{}, backlog)
|
||||
for i := range backlog {
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {
|
||||
mu.Lock()
|
||||
order = append(order, i)
|
||||
mu.Unlock()
|
||||
done <- struct{}{}
|
||||
})
|
||||
}
|
||||
|
||||
if got := turns.Queued("s1"); got != backlog {
|
||||
t.Fatalf("Queued() = %d, want all %d held", got, backlog)
|
||||
}
|
||||
close(release)
|
||||
|
||||
for range backlog {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("queued turns did not all run")
|
||||
}
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(order) != backlog {
|
||||
t.Fatalf("ran %d turns, want %d", len(order), backlog)
|
||||
}
|
||||
for i, got := range order {
|
||||
if got != i {
|
||||
t.Fatalf("turn %d ran at position %d; the queue is not FIFO", got, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTurnsRunSeriallyWithinSession covers the queueing half of the
|
||||
// contract: replies must not interleave for one conversation.
|
||||
func TestTurnsRunSeriallyWithinSession(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
var mu sync.Mutex
|
||||
var concurrent, peak int
|
||||
done := make(chan struct{}, 3)
|
||||
|
||||
for range 3 {
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) {
|
||||
mu.Lock()
|
||||
concurrent++
|
||||
if concurrent > peak {
|
||||
peak = concurrent
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
concurrent--
|
||||
mu.Unlock()
|
||||
done <- struct{}{}
|
||||
})
|
||||
}
|
||||
for range 3 {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("turns did not complete")
|
||||
}
|
||||
}
|
||||
if peak != 1 {
|
||||
t.Errorf("peak concurrency = %d, want 1; turns in one session overlapped", peak)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionsAreIndependent checks that one busy session cannot stall
|
||||
// another, and that Stop is scoped to the session that asked for it.
|
||||
func TestSessionsAreIndependent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
busy := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
defer close(release)
|
||||
|
||||
turns.Submit(context.Background(), "busy", func(context.Context) {
|
||||
close(busy)
|
||||
<-release
|
||||
})
|
||||
<-busy
|
||||
|
||||
ran := make(chan struct{})
|
||||
turns.Submit(context.Background(), "other", func(context.Context) { close(ran) })
|
||||
select {
|
||||
case <-ran:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a second session was blocked behind the first")
|
||||
}
|
||||
|
||||
if cancelled, _ := turns.Stop("other"); cancelled {
|
||||
t.Error("Stop cancelled a turn in an idle session")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopAfterCompletionIsInert guards the retraction in runOne: a stale
|
||||
// cancel left behind by a finished turn would kill the next one.
|
||||
func TestStopAfterCompletionIsInert(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
done := make(chan struct{})
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) { close(done) })
|
||||
<-done
|
||||
waitFor(t, func() bool { return !turns.Running("s1") })
|
||||
|
||||
if cancelled, dropped := turns.Stop("s1"); cancelled || dropped != 0 {
|
||||
t.Errorf("Stop() = (%v, %d) on an idle session, want (false, 0)", cancelled, dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPanicInTurnDoesNotKillLane covers the recovery in runOne. Losing the
|
||||
// lane goroutine would leave the session permanently unable to reply.
|
||||
func TestPanicInTurnDoesNotKillLane(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
turns := testTurns(t)
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) { panic("boom") })
|
||||
|
||||
ran := make(chan struct{})
|
||||
turns.Submit(context.Background(), "s1", func(context.Context) { close(ran) })
|
||||
select {
|
||||
case <-ran:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the lane died with the panicking turn")
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("condition not met within timeout")
|
||||
}
|
||||
Reference in new issue
Block a user