From 561e56b70ae6d71cbb0a1a86e3bef151b768a201 Mon Sep 17 00:00:00 2001 From: Sam Catlow Date: Wed, 29 Jul 2026 16:41:55 +1000 Subject: [PATCH] feat: make /stop cancel the running chat turn /stop could not reach a running turn, and the reason was worse than a missing cancel. go-telegram/bot runs one update worker that calls handlers synchronously (defaultWorkers = 1, never overridden), and the chat turn ran on it. While a turn streamed -- or sat in a shell tool -- the worker was blocked and /stop sat unread in the update channel until the turn it was meant to kill had already finished. Add gateway.Turns: a per-session serial lane that turns are submitted to and that the event loop never waits on. The worker is free again in microseconds, so /stop is serviced immediately; ordering comes from the lane being serial rather than from the loop being blocked. - Bare /stop cancels the running turn and clears the backlog behind it, with no approval step. Approval guards destructive commands, and gating the brake behind a confirmation defeats it. /stop keeps the existing process-termination behaviour and its approval. - Cancellation propagates through the turn context, so it reaches the model stream and any tool underneath -- a shell command built with exec.CommandContext dies with the turn. - Queues are unbounded and never shed. A dropped message is a silent failure that surfaces later as an agent that ignored something. - Turns are recovered per-lane: the channel's panic middleware wraps the event-loop goroutine, which no longer runs the turn. Also raise the chat step budget. It was hardcoded at 8 -- roughly one read, one grep and two edits -- so turns hit the cap and stopped mid-task with no indication they had been cut off. It is now chat.max_steps, defaulting to 100. Interruption is /stop's job. --- cmd/archied/main.go | 16 +- cmd/archied/main_test.go | 17 +- cmd/archied/telegram_setup.go | 2 +- config.example.toml | 8 + internal/channels/telegram/dangerous.go | 73 ++++- internal/channels/telegram/dangerous_test.go | 63 +++- internal/channels/telegram/telegram.go | 67 +++- internal/config/config.go | 9 + internal/gateway/turns.go | 229 ++++++++++++++ internal/gateway/turns_test.go | 315 +++++++++++++++++++ 10 files changed, 765 insertions(+), 34 deletions(-) create mode 100644 internal/gateway/turns.go create mode 100644 internal/gateway/turns_test.go diff --git a/cmd/archied/main.go b/cmd/archied/main.go index 348f2519b..f5e571570 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -71,7 +71,15 @@ func main() { } const ( - defaultChatMaxSteps = 8 + // defaultChatMaxSteps bounds the model/tool round-trips in one chat + // turn when [config.ChatConfig.MaxSteps] is unset. + // + // It was 8, which is roughly one read, one grep and a couple of edits + // -- less than a single small change to a Go package costs. Turns hit + // the cap and stopped mid-task with no indication they had been cut + // off. Interruption is /stop's job; this only has to stop a genuine + // runaway, so it is set well above real work. + defaultChatMaxSteps = 100 packagedGatewayChangelogPath = "/usr/share/archie/CHANGELOG.archied.md" packagedRuntimeChangelogPath = "/usr/share/archie/CHANGELOG.archie.md" ) @@ -87,15 +95,19 @@ var ( func chatGenerateOptions( messages []chat.Message, registry *tools.Registry, + maxSteps int, ) (core.GenerateOptions, error) { toolSet, err := agentexec.BuildToolSet(registry) if err != nil { return core.GenerateOptions{}, err } + if maxSteps <= 0 { + maxSteps = defaultChatMaxSteps + } return core.GenerateOptions{ Messages: messages, Tools: toolSet, - MaxSteps: defaultChatMaxSteps, + MaxSteps: maxSteps, }, nil } diff --git a/cmd/archied/main_test.go b/cmd/archied/main_test.go index 604f344a1..7e3e73214 100644 --- a/cmd/archied/main_test.go +++ b/cmd/archied/main_test.go @@ -38,7 +38,7 @@ func TestChatGenerateOptionsIncludesToolsAndMultipleSteps(t *testing.T) { } messages := []chat.Message{{Role: chat.RoleUser, Content: "remember this"}} - options, err := chatGenerateOptions(messages, registry) + options, err := chatGenerateOptions(messages, registry, 0) if err != nil { t.Fatal(err) } @@ -64,11 +64,24 @@ func TestChatGenerateOptionsReportsInvalidToolSchemas(t *testing.T) { }); err != nil { t.Fatal(err) } - if _, err := chatGenerateOptions(nil, registry); err == nil { + if _, err := chatGenerateOptions(nil, registry, 0); err == nil { t.Fatal("chatGenerateOptions() error = nil, want invalid tool schema error") } } +// TestChatGenerateOptionsHonoursConfiguredMaxSteps keeps the step budget +// deployment-controlled. A cap that cannot be raised without a rebuild is +// the situation this replaced. +func TestChatGenerateOptionsHonoursConfiguredMaxSteps(t *testing.T) { + options, err := chatGenerateOptions(nil, tools.NewRegistry(), 250) + if err != nil { + t.Fatal(err) + } + if options.MaxSteps != 250 { + t.Errorf("MaxSteps = %d, want the configured 250", options.MaxSteps) + } +} + func TestGatewaysStartOnlyAfterCapabilityHostAndDaemonStartup(t *testing.T) { source, err := os.ReadFile("main.go") if err != nil { diff --git a/cmd/archied/telegram_setup.go b/cmd/archied/telegram_setup.go index f6945fca6..3d0e72655 100644 --- a/cmd/archied/telegram_setup.go +++ b/cmd/archied/telegram_setup.go @@ -180,7 +180,7 @@ func makeTelegramLLMResponder(ctx context.Context, tg *telegram.Gateway, s teleg compressed = append(compressed, gateway.CompressedMessage{Role: role, Content: h.Text}) } view := gateway.CompressHistory(compressed, gateway.DefaultCompressionConfig()) - options, err := chatGenerateOptions(nil, s.ToolReg) + options, err := chatGenerateOptions(nil, s.ToolReg, s.Cfg.Chat.MaxSteps) if err != nil { return "", fmt.Errorf("build chat tools: %w", err) } diff --git a/config.example.toml b/config.example.toml index fa1b0da7d..b336d7f85 100644 --- a/config.example.toml +++ b/config.example.toml @@ -55,9 +55,17 @@ class = "ollama" # shell, grep, find). Unset, those tools are not registered at all. Set it and # the agent can read, modify and execute inside that directory with the same # reach the daemon process has -- choose it deliberately. +# +# `max_steps` caps the model/tool round-trips in a single chat turn +# (default 100). This is a task budget, not a safety limit: one question +# about a codebase easily costs a read, several greps and a few edits, and +# a turn that runs out stops mid-task looking like it gave up. To interrupt +# a turn, send /stop -- that cancels it immediately and clears anything +# queued behind it. # [chat] # operator = "Ada Lovelace" # workspace = "/var/lib/archie/workspace" +# max_steps = 100 # models = [ # "openai/gpt-5.6-sol", # "openrouter/openai/gpt-5.6-sol", diff --git a/internal/channels/telegram/dangerous.go b/internal/channels/telegram/dangerous.go index e4977558a..f97ada7ff 100644 --- a/internal/channels/telegram/dangerous.go +++ b/internal/channels/telegram/dangerous.go @@ -12,6 +12,8 @@ import ( "github.com/go-telegram/bot" "github.com/go-telegram/bot/models" + + "github.com/samcharles93/archie-core/internal/gateway" ) // DangerousCommandAuthority is the sandbox/process/filesystem authority @@ -405,25 +407,74 @@ func rollbackApprovalText(num int) string { // ── /stop handler ────────────────────────────────────────────── -// stopHandler serves /stop: terminate a background process after approval. -func (g *Gateway) stopHandler() bot.HandlerFunc { +// stopCurrentTurn cancels whatever this conversation is currently doing. +// +// Cancellation propagates through the turn's context, so it reaches the +// model stream and any tool running underneath it -- a shell command built +// with exec.CommandContext is killed with the turn, which is the case that +// matters. Whatever the reply had already streamed stays on screen: it is +// the record of what Archie did before being stopped. +func (g *Gateway) stopCurrentTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) { + var cancelled bool + var dropped int + + // turns is only built by launch, so a gateway that has never started + // simply has nothing to stop. Report that rather than failing. + if g.turns != nil && router != nil { + session, err := router.ResolveSessionKey(ctx, gateway.Message{ + ChannelID: fmt.Sprintf("%d", msg.Chat.ID), + ThreadID: threadIDString(msg.MessageThreadID), + From: msg.From.Username, + Text: msg.Text, + }) + if err != nil { + g.log.Error("resolve session for stop", "error", err) + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.") + return + } + cancelled, dropped = g.turns.Stop(session) + g.log.Info("stop requested", "session", session, "cancelled", cancelled, "dropped", dropped) + } + + switch { + case cancelled && dropped > 0: + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, + fmt.Sprintf("🛑 Stopped. %d queued message(s) discarded.", dropped)) + case cancelled: + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "🛑 Stopped.") + case dropped > 0: + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, + fmt.Sprintf("🛑 Nothing was running. %d queued message(s) discarded.", dropped)) + default: + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, + "Nothing is running.\n\nUse /stop to terminate a background process.") + } +} + +// stopHandler serves /stop. +// +// Bare /stop is the emergency brake: it cancels the conversation's running +// turn immediately, with no approval step. Approval exists to protect +// against destructive commands, and stopping is the opposite -- gating it +// behind a confirmation round-trip would defeat the entire point. +// +// /stop keeps the original behaviour of terminating a named +// background process, which is destructive and still requires approval. +func (g *Gateway) stopHandler(router *gateway.Router) bot.HandlerFunc { return func(ctx context.Context, b *bot.Bot, update *models.Update) { msg, ok := g.authorizedMessage(ctx, b, update) if !ok { return } - if g.Dangerous == nil { - g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.") + + rest := strings.TrimSpace(restAfterTelegram(msg.Text, "/stop", "")) + if rest == "" { + g.stopCurrentTurn(ctx, b, msg, router) return } - rest := restAfterTelegram(msg.Text, "/stop", "") - rest = strings.TrimSpace(rest) - - if rest == "" { - g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, - "Usage: /stop \n\nTerminate a running background process. "+ - "This requires approval because it may discard in-progress work.") + if g.Dangerous == nil { + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.") return } diff --git a/internal/channels/telegram/dangerous_test.go b/internal/channels/telegram/dangerous_test.go index ae26ca105..1dd41ede8 100644 --- a/internal/channels/telegram/dangerous_test.go +++ b/internal/channels/telegram/dangerous_test.go @@ -11,6 +11,8 @@ import ( "time" "github.com/go-telegram/bot/models" + + "github.com/samcharles93/archie-core/internal/gateway" ) // ── DangerousCommandAuthority stub ───────────────────────────── @@ -303,7 +305,7 @@ func TestStopRequiresApproval(t *testing.T) { g.Dangerous = stub b, requests := newTelegramTestBot(t) - g.stopHandler()(context.Background(), b, &models.Update{ + g.stopHandler(nil)(context.Background(), b, &models.Update{ Message: &models.Message{ From: &models.User{ID: allowedUserID}, Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate}, @@ -333,7 +335,7 @@ func TestStopApprovalExecutes(t *testing.T) { b, requests := newTelegramTestBot(t) // Step 1: Trigger the approval prompt. - g.stopHandler()(context.Background(), b, &models.Update{ + g.stopHandler(nil)(context.Background(), b, &models.Update{ Message: &models.Message{ From: &models.User{ID: allowedUserID}, Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate}, @@ -367,14 +369,18 @@ func TestStopApprovalExecutes(t *testing.T) { } } -func TestStopNoArgument(t *testing.T) { +// TestStopWithNoArgumentDoesNotRequireApproval is the emergency-brake +// contract. Bare /stop targets the running chat turn, so it must act +// immediately -- if it ever renders an approval prompt again, the brake has +// been put behind a confirmation round-trip and no longer works. +func TestStopWithNoArgumentDoesNotRequireApproval(t *testing.T) { const allowedUserID = int64(42) stub := &dangerousStub{} g := New("1:test", "", "", []int64{allowedUserID}, slog.Default()) g.Dangerous = stub b, requests := newTelegramTestBot(t) - g.stopHandler()(context.Background(), b, &models.Update{ + g.stopHandler(nil)(context.Background(), b, &models.Update{ Message: &models.Message{ From: &models.User{ID: allowedUserID}, Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate}, @@ -383,8 +389,51 @@ func TestStopNoArgument(t *testing.T) { }) text := messageText(*requests, 0) - if !strings.Contains(text, "Usage: /stop") { - t.Errorf("expected usage message, got: %s", text) + if strings.Contains(text, "Dangerous command") { + t.Errorf("bare /stop asked for approval: %s", text) + } + if !strings.Contains(text, "Nothing is running") { + t.Errorf("expected an idle report, got: %s", text) + } + if len(stub.stopCalls) != 0 { + t.Errorf("bare /stop terminated a named process: %v", stub.stopCalls) + } +} + +// TestStopCancelsRunningChatTurn covers the whole point of the command: +// a turn in flight is cancelled, and the queue behind it is discarded. +func TestStopCancelsRunningChatTurn(t *testing.T) { + const allowedUserID = int64(42) + g := New("1:test", "", "", []int64{allowedUserID}, slog.Default()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + g.turns = gateway.NewTurns(slog.Default()) + + // stopCurrentTurn resolves the session through the router; with none + // wired, drive the lane directly and assert on the lane's state. + const session = "chat:7" + started := make(chan struct{}) + stopped := make(chan struct{}) + g.turns.Submit(ctx, session, func(turnCtx context.Context) { + close(started) + <-turnCtx.Done() + close(stopped) + }) + <-started + g.turns.Submit(ctx, session, func(context.Context) {}) + + cancelled, dropped := g.turns.Stop(session) + if !cancelled { + t.Error("the running turn was not cancelled") + } + if dropped != 1 { + t.Errorf("dropped = %d, want 1", dropped) + } + select { + case <-stopped: + case <-time.After(2 * time.Second): + t.Fatal("the turn did not observe cancellation") } } @@ -851,7 +900,7 @@ func TestStopNotConfigured(t *testing.T) { // Dangerous is nil. b, requests := newTelegramTestBot(t) - g.stopHandler()(context.Background(), b, &models.Update{ + g.stopHandler(nil)(context.Background(), b, &models.Update{ Message: &models.Message{ From: &models.User{ID: allowedUserID}, Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate}, diff --git a/internal/channels/telegram/telegram.go b/internal/channels/telegram/telegram.go index 70963f344..bb5dfc15f 100644 --- a/internal/channels/telegram/telegram.go +++ b/internal/channels/telegram/telegram.go @@ -6,6 +6,7 @@ package telegram import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "net/http" @@ -71,6 +72,11 @@ type Gateway struct { dangerousActions map[string]dangerousAction permanentApprovals []permanentApproval + // turns serialises chat turns per session off the update worker and + // makes the running one cancellable by /stop. Rebuilt on every launch + // so a restart abandons in-flight turns with the old bot instance. + turns *gateway.Turns + log *slog.Logger bot *bot.Bot webhookCancel context.CancelFunc @@ -162,6 +168,11 @@ func (g *Gateway) Start(ctx context.Context, router *gateway.Router) error { // launch builds one bot instance, registers handlers and begins receiving. // It returns as soon as delivery is running; the caller owns ctx. func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot, error) { + // Turns are per-launch. Each queued turn carries the update handler's + // context, which is this launch's, so a /restart cancels everything + // still running under the outgoing bot instance. + g.turns = gateway.NewTurns(g.log) + opts := []bot.Option{ bot.WithErrorsHandler(func(err error) { g.log.Error("telegram pipeline error", "error", err) @@ -195,7 +206,7 @@ func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot, b.RegisterHandler(bot.HandlerTypeMessageText, "/help", bot.MatchTypeExact, g.helpHandler()) b.RegisterHandler(bot.HandlerTypeMessageText, "/restart", bot.MatchTypeExact, g.restartHandler()) b.RegisterHandler(bot.HandlerTypeMessageText, "/rollback", bot.MatchTypePrefix, g.rollbackHandler()) - b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler()) + b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler(router)) b.RegisterHandler(bot.HandlerTypeMessageText, "/approve", bot.MatchTypeExact, g.approveHandler()) b.RegisterHandler(bot.HandlerTypeMessageText, "/deny", bot.MatchTypeExact, g.denyHandler()) @@ -434,29 +445,63 @@ func (g *Gateway) defaultHandler(router *gateway.Router) bot.HandlerFunc { g.handlePersonalityCommand(ctx, b, msg, router) return } + g.submitTurn(ctx, b, msg, router) + } +} + +// submitTurn hands the message to its session's turn lane and returns at +// once, leaving the bot's single update worker free. +// +// The turn must not run on the caller's goroutine. go-telegram/bot serves +// updates from one worker that invokes handlers synchronously, so a turn +// running here would block delivery of every later update -- including the +// /stop meant to cancel it, which would sit unread until the turn it was +// aimed at had already finished. +func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) { + gm := gateway.Message{ + ChannelID: fmt.Sprintf("%d", msg.Chat.ID), + ThreadID: threadIDString(msg.MessageThreadID), + From: msg.From.Username, + Text: msg.Text, + } + + // The lane key must be the session, so that /stop -- which resolves + // the same key -- reaches the turn the sender is actually watching. + session, err := router.ResolveSessionKey(ctx, gm) + if err != nil { + g.log.Error("resolve session for turn", "error", err) + g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.") + return + } + + chatID, threadID := msg.Chat.ID, msg.MessageThreadID + g.turns.Submit(ctx, session, func(turnCtx context.Context) { // reply appears as it is written; the typing indicator covers the // gap before the first token and any non-streaming path. - stopTyping := g.startTyping(ctx, b, msg.Chat.ID, msg.MessageThreadID) - draft := g.newDraft(b, msg.Chat.ID, msg.MessageThreadID) + stopTyping := g.startTyping(turnCtx, b, chatID, threadID) + draft := g.newDraft(b, chatID, threadID) // If it starts with / but wasn't matched by a registered // handler, it's unknown -- let the router handle it (which // will say "unrecognized"). - reply, err := router.RouteStream(ctx, gateway.Message{ - ChannelID: fmt.Sprintf("%d", msg.Chat.ID), - ThreadID: threadIDString(msg.MessageThreadID), - From: msg.From.Username, - Text: msg.Text, - }, draft.onDelta) + reply, err := router.RouteStream(turnCtx, gm, draft.onDelta) stopTyping() + + // A cancelled turn is a /stop, not a fault. The stop handler has + // already acknowledged it, and turnCtx is dead, so there is + // nothing useful left to send from here. + if errors.Is(err, context.Canceled) { + g.log.Info("chat turn stopped", "session", session) + return + } if err != nil { g.log.Error("route failed", "error", err) return } if reply != "" { - g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, reply) + g.sendMessage(turnCtx, b, chatID, threadID, reply) } - } + }) } // ── helpers ────────────────────────────────────────────────── diff --git a/internal/config/config.go b/internal/config/config.go index bad5ec86c..3c3594b2a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -491,6 +491,15 @@ type ChatConfig struct { // they read, write and execute, so the directory must be a deliberate // choice rather than whatever the daemon happens to start in. Workspace string `toml:"workspace" yaml:"workspace"` + // MaxSteps caps how many model/tool round-trips one chat turn may take + // before the runtime stops and returns what it has. Zero uses the + // default. + // + // This is a real task budget, not a safety limit: a single question + // about a codebase routinely costs a read, several greps and a handful + // of edits, and running out mid-turn looks to the user like the agent + // simply gave up. Use /stop to interrupt, not a small step cap. + MaxSteps int `toml:"max_steps" yaml:"max_steps"` // Models is the optional interactive-chat model catalog. When empty, // chat falls back to the distinct model references assigned to workflow // roles in the top-level [models] table. diff --git a/internal/gateway/turns.go b/internal/gateway/turns.go new file mode 100644 index 000000000..f6f92e233 --- /dev/null +++ b/internal/gateway/turns.go @@ -0,0 +1,229 @@ +package gateway + +import ( + "context" + "log/slog" + "runtime/debug" + "sync" +) + +// Turns serialises chat turns per session and keeps the running turn +// cancellable. +// +// Channel event loops are typically single-threaded -- go-telegram/bot, +// for one, runs a single worker that calls handlers synchronously. Running +// a turn on that goroutine blocks delivery of every subsequent update, +// including the command meant to stop it: /stop would sit unread in the +// update channel until the turn it was meant to kill had already finished. +// +// Submitting to Turns hands the work to a per-session goroutine and returns +// immediately, so the event loop stays free and control commands are always +// serviceable. Ordering is preserved by the lane being serial, not by the +// event loop being blocked. +// +// Queues are unbounded and never shed. A dropped message is a silent +// failure that surfaces much later as an agent that ignored something, +// which costs far more to diagnose than the memory a backlog occupies. +// [Turns.Stop] is the way a backlog is cleared -- deliberately, by the +// person who created it. +type Turns struct { + log *slog.Logger + + mu sync.Mutex + lanes map[string]*lane +} + +// queued is one pending turn: the work, already bound to its own +// cancellable context, and the cancel that releases it. +// +// The context is derived when the turn is submitted rather than when it +// starts running, which means a turn has a live cancel handle for its whole +// time in the backlog. Stop can therefore cancel queued turns rather than +// merely dropping them, and each turn stays tied to the caller that asked +// for it -- it dies when that caller's context does. +type queued struct { + run func() + cancel context.CancelFunc +} + +// lane is one session's serial worker. +// +// A single mutex guards the backlog, the running turn's cancel, and whether +// a worker is live, so that Stop observes them together. Splitting them +// would leave a window where a turn finishes between the backlog being +// cleared and the cancel being called, letting a queued turn start after a +// stop. +type lane struct { + mu sync.Mutex + pending []queued + // cancel stops the turn currently running in this lane. It is nil + // whenever the lane is idle. + cancel context.CancelFunc + // working reports whether a worker goroutine is draining this lane. + // The worker exits when the backlog empties, so this is what tells + // Submit whether to start a new one. + working bool +} + +// NewTurns returns an idle dispatcher. +func NewTurns(log *slog.Logger) *Turns { + return &Turns{log: log, lanes: make(map[string]*lane)} +} + +// Submit schedules run for the given session and returns without waiting +// for it. run receives a context derived from ctx that [Turns.Stop] can +// cancel. +// +// Submit never blocks and never rejects: messages queue. +func (t *Turns) Submit(ctx context.Context, session string, run func(context.Context)) { + t.mu.Lock() + l, ok := t.lanes[session] + if !ok { + l = &lane{} + t.lanes[session] = l + } + t.mu.Unlock() + + turnCtx, cancel := context.WithCancel(ctx) + + l.mu.Lock() + l.pending = append(l.pending, queued{run: func() { run(turnCtx) }, cancel: cancel}) + depth := len(l.pending) + start := !l.working + if start { + l.working = true + } + l.mu.Unlock() + + if start { + go l.serve(t.log.With("session", session)) + } + if depth > 1 { + t.log.Info("chat turn queued", "session", session, "queue_depth", depth) + } +} + +// Stop cancels the session's running turn and discards everything queued +// behind it. It reports whether a turn was actually cancelled and how many +// queued turns were dropped. +// +// Dropping the queue is intentional. "Stop" means stop, not "stop this one +// and immediately begin the next thing I said" -- starting the queued turn +// would look to the sender exactly like the stop had failed. +func (t *Turns) Stop(session string) (cancelled bool, dropped int) { + l := t.lane(session) + if l == nil { + return false, 0 + } + + l.mu.Lock() + defer l.mu.Unlock() + + dropped = len(l.pending) + for i, q := range l.pending { + q.cancel() + l.pending[i] = queued{} + } + l.pending = nil + if l.cancel != nil { + l.cancel() + cancelled = true + } + return cancelled, dropped +} + +// Running reports whether the session currently has a turn in flight. +func (t *Turns) Running(session string) bool { + l := t.lane(session) + if l == nil { + return false + } + l.mu.Lock() + defer l.mu.Unlock() + return l.cancel != nil +} + +// Queued reports how many turns are waiting behind the running one. +func (t *Turns) Queued(session string) int { + l := t.lane(session) + if l == nil { + return 0 + } + l.mu.Lock() + defer l.mu.Unlock() + return len(l.pending) +} + +// lane returns the session's lane, or nil if it has never been used. +func (t *Turns) lane(session string) *lane { + t.mu.Lock() + defer t.mu.Unlock() + return t.lanes[session] +} + +// serve drains the lane's backlog one turn at a time and exits once it is +// empty. Submit starts a fresh worker when more arrives, so an idle session +// costs no goroutine. +func (l *lane) serve(log *slog.Logger) { + for { + next, ok := l.begin() + if !ok { + return + } + l.runOne(next, log) + } +} + +// begin pops the oldest queued turn and publishes its cancel in the same +// critical section, reporting false when the backlog is empty -- at which +// point it also retires the worker. +// +// The pop and the publish must happen together. Were the turn popped first +// and its cancel published after, a Stop landing in between would see an +// empty backlog and no running turn, report that nothing was happening, and +// let the turn it should have killed start regardless. +func (l *lane) begin() (queued, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + if len(l.pending) == 0 { + // Retire under the same lock Submit uses to decide whether a + // worker is live, so a turn arriving now either starts a new + // worker or is picked up before this one retires -- never + // neither, which would strand the backlog forever. + l.working = false + return queued{}, false + } + + next := l.pending[0] + // Clear the vacated slot so a completed turn's closure -- and whatever + // it captured -- does not stay reachable through the backing array. + l.pending[0] = queued{} + l.pending = l.pending[1:] + + l.cancel = next.cancel + return next, true +} + +// runOne executes one turn, always retracting its cancel afterwards so a +// later Stop cannot cancel an already-finished turn. +// +// Panics are contained here rather than by the channel's own recovery +// middleware: that middleware wraps the event-loop goroutine, and the turn +// no longer runs on it. Without this, one bad turn would take the lane's +// goroutine down and the session would silently stop answering forever. +func (l *lane) runOne(next queued, log *slog.Logger) { + defer next.cancel() + + defer func() { + l.mu.Lock() + l.cancel = nil + l.mu.Unlock() + + if r := recover(); r != nil { + log.Error("chat turn panicked", "panic", r, "stack", string(debug.Stack())) + } + }() + + next.run() +} diff --git a/internal/gateway/turns_test.go b/internal/gateway/turns_test.go new file mode 100644 index 000000000..c58617d42 --- /dev/null +++ b/internal/gateway/turns_test.go @@ -0,0 +1,315 @@ +package gateway_test + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/samcharles93/archie-core/internal/gateway" +) + +func testTurns(t *testing.T) *gateway.Turns { + t.Helper() + return gateway.NewTurns(slog.New(slog.NewTextHandler(io.Discard, nil))) +} + +// TestSubmitDoesNotBlockCaller is the whole reason this type exists: the +// channel event loop must stay free while a turn runs, or the command that +// stops the turn can never be delivered. +func TestSubmitDoesNotBlockCaller(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + release := make(chan struct{}) + started := make(chan struct{}) + + turns.Submit(context.Background(), "s1", func(context.Context) { + close(started) + <-release + }) + <-started + defer close(release) + + returned := make(chan struct{}) + go func() { + turns.Submit(context.Background(), "s1", func(context.Context) {}) + close(returned) + }() + select { + case <-returned: + case <-time.After(2 * time.Second): + t.Fatal("Submit blocked while a turn was running; the event loop would be stuck") + } +} + +func TestStopCancelsRunningTurn(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + started := make(chan struct{}) + var cause atomic.Value + + turns.Submit(context.Background(), "s1", func(ctx context.Context) { + close(started) + <-ctx.Done() + cause.Store(ctx.Err()) + }) + <-started + + cancelled, dropped := turns.Stop("s1") + if !cancelled { + t.Error("Stop reported no running turn") + } + if dropped != 0 { + t.Errorf("dropped = %d, want 0", dropped) + } + + waitFor(t, func() bool { return cause.Load() != nil }) + if err, _ := cause.Load().(error); !errors.Is(err, context.Canceled) { + t.Errorf("turn context error = %v, want context.Canceled", err) + } +} + +// TestStopDropsQueuedTurns guards the "stop means stop" rule. If the +// queued turn ran after cancellation, the sender would see a new reply +// begin immediately and conclude the stop had failed. +func TestStopDropsQueuedTurns(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + started := make(chan struct{}) + release := make(chan struct{}) + var queuedRan atomic.Bool + + turns.Submit(context.Background(), "s1", func(context.Context) { + close(started) + <-release + }) + <-started + + for range 2 { + turns.Submit(context.Background(), "s1", func(context.Context) { queuedRan.Store(true) }) + } + + cancelled, dropped := turns.Stop("s1") + if !cancelled || dropped != 2 { + t.Fatalf("Stop() = (%v, %d), want (true, 2)", cancelled, dropped) + } + close(release) + + // Give the lane time to pull anything that survived the drain. + time.Sleep(100 * time.Millisecond) + if queuedRan.Load() { + t.Error("a queued turn ran after /stop") + } +} + +// TestStopLeavesTheCallersContextAlone checks that Stop only reaches the +// contexts Turns derived. Callers pass in the channel's long-lived run +// context, so cancelling that instead of the turn's own would take the +// whole gateway down every time someone sent /stop. +func TestStopLeavesTheCallersContextAlone(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + callerCtx, callerCancel := context.WithCancel(context.Background()) + defer callerCancel() + + started := make(chan struct{}) + turns.Submit(callerCtx, "s1", func(ctx context.Context) { + close(started) + <-ctx.Done() + }) + <-started + turns.Submit(callerCtx, "s1", func(context.Context) {}) + + if cancelled, dropped := turns.Stop("s1"); !cancelled || dropped != 1 { + t.Fatalf("Stop() = (%v, %d), want (true, 1)", cancelled, dropped) + } + if err := callerCtx.Err(); err != nil { + t.Errorf("Stop cancelled the caller's context: %v", err) + } + + // The lane must still be usable afterwards. + ran := make(chan struct{}) + turns.Submit(callerCtx, "s1", func(context.Context) { close(ran) }) + select { + case <-ran: + case <-time.After(2 * time.Second): + t.Fatal("the session stopped accepting turns after /stop") + } +} + +// TestDeepBacklogIsKeptAndOrdered is the no-shedding contract. A message +// that is silently dropped surfaces much later as an agent that ignored +// something, which is far more expensive to diagnose than holding the +// backlog. Every message queues, and they run in the order they arrived. +func TestDeepBacklogIsKeptAndOrdered(t *testing.T) { + t.Parallel() + + const backlog = 100 + + turns := testTurns(t) + started := make(chan struct{}) + release := make(chan struct{}) + + turns.Submit(context.Background(), "s1", func(context.Context) { + close(started) + <-release + }) + <-started + + var mu sync.Mutex + var order []int + done := make(chan struct{}, backlog) + for i := range backlog { + turns.Submit(context.Background(), "s1", func(context.Context) { + mu.Lock() + order = append(order, i) + mu.Unlock() + done <- struct{}{} + }) + } + + if got := turns.Queued("s1"); got != backlog { + t.Fatalf("Queued() = %d, want all %d held", got, backlog) + } + close(release) + + for range backlog { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("queued turns did not all run") + } + } + + mu.Lock() + defer mu.Unlock() + if len(order) != backlog { + t.Fatalf("ran %d turns, want %d", len(order), backlog) + } + for i, got := range order { + if got != i { + t.Fatalf("turn %d ran at position %d; the queue is not FIFO", got, i) + } + } +} + +// TestTurnsRunSeriallyWithinSession covers the queueing half of the +// contract: replies must not interleave for one conversation. +func TestTurnsRunSeriallyWithinSession(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + var mu sync.Mutex + var concurrent, peak int + done := make(chan struct{}, 3) + + for range 3 { + turns.Submit(context.Background(), "s1", func(context.Context) { + mu.Lock() + concurrent++ + if concurrent > peak { + peak = concurrent + } + mu.Unlock() + + time.Sleep(20 * time.Millisecond) + + mu.Lock() + concurrent-- + mu.Unlock() + done <- struct{}{} + }) + } + for range 3 { + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("turns did not complete") + } + } + if peak != 1 { + t.Errorf("peak concurrency = %d, want 1; turns in one session overlapped", peak) + } +} + +// TestSessionsAreIndependent checks that one busy session cannot stall +// another, and that Stop is scoped to the session that asked for it. +func TestSessionsAreIndependent(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + busy := make(chan struct{}) + release := make(chan struct{}) + defer close(release) + + turns.Submit(context.Background(), "busy", func(context.Context) { + close(busy) + <-release + }) + <-busy + + ran := make(chan struct{}) + turns.Submit(context.Background(), "other", func(context.Context) { close(ran) }) + select { + case <-ran: + case <-time.After(2 * time.Second): + t.Fatal("a second session was blocked behind the first") + } + + if cancelled, _ := turns.Stop("other"); cancelled { + t.Error("Stop cancelled a turn in an idle session") + } +} + +// TestStopAfterCompletionIsInert guards the retraction in runOne: a stale +// cancel left behind by a finished turn would kill the next one. +func TestStopAfterCompletionIsInert(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + done := make(chan struct{}) + turns.Submit(context.Background(), "s1", func(context.Context) { close(done) }) + <-done + waitFor(t, func() bool { return !turns.Running("s1") }) + + if cancelled, dropped := turns.Stop("s1"); cancelled || dropped != 0 { + t.Errorf("Stop() = (%v, %d) on an idle session, want (false, 0)", cancelled, dropped) + } +} + +// TestPanicInTurnDoesNotKillLane covers the recovery in runOne. Losing the +// lane goroutine would leave the session permanently unable to reply. +func TestPanicInTurnDoesNotKillLane(t *testing.T) { + t.Parallel() + + turns := testTurns(t) + turns.Submit(context.Background(), "s1", func(context.Context) { panic("boom") }) + + ran := make(chan struct{}) + turns.Submit(context.Background(), "s1", func(context.Context) { close(ran) }) + select { + case <-ran: + case <-time.After(2 * time.Second): + t.Fatal("the lane died with the panicking turn") + } +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("condition not met within timeout") +}