feat: make /stop cancel the running chat turn

/stop could not reach a running turn, and the reason was worse than a
missing cancel. go-telegram/bot runs one update worker that calls
handlers synchronously (defaultWorkers = 1, never overridden), and the
chat turn ran on it. While a turn streamed -- or sat in a shell tool --
the worker was blocked and /stop sat unread in the update channel until
the turn it was meant to kill had already finished.

Add gateway.Turns: a per-session serial lane that turns are submitted to
and that the event loop never waits on. The worker is free again in
microseconds, so /stop is serviced immediately; ordering comes from the
lane being serial rather than from the loop being blocked.

- Bare /stop cancels the running turn and clears the backlog behind it,
  with no approval step. Approval guards destructive commands, and
  gating the brake behind a confirmation defeats it. /stop <name> keeps
  the existing process-termination behaviour and its approval.
- Cancellation propagates through the turn context, so it reaches the
  model stream and any tool underneath -- a shell command built with
  exec.CommandContext dies with the turn.
- Queues are unbounded and never shed. A dropped message is a silent
  failure that surfaces later as an agent that ignored something.
- Turns are recovered per-lane: the channel's panic middleware wraps the
  event-loop goroutine, which no longer runs the turn.

Also raise the chat step budget. It was hardcoded at 8 -- roughly one
read, one grep and two edits -- so turns hit the cap and stopped
mid-task with no indication they had been cut off. It is now
chat.max_steps, defaulting to 100. Interruption is /stop's job.
This commit is contained in:
sam committed 2026-07-29 16:41:55 +10:00
1 parent e03cef17c9
commit 561e56b70a
10 files changed
+765 -34

No files matched your search

+14 -2
View File
@@ -71,7 +71,15 @@ func main() {
}
const (
defaultChatMaxSteps = 8
// defaultChatMaxSteps bounds the model/tool round-trips in one chat
// turn when [config.ChatConfig.MaxSteps] is unset.
//
// It was 8, which is roughly one read, one grep and a couple of edits
// -- less than a single small change to a Go package costs. Turns hit
// the cap and stopped mid-task with no indication they had been cut
// off. Interruption is /stop's job; this only has to stop a genuine
// runaway, so it is set well above real work.
defaultChatMaxSteps = 100
packagedGatewayChangelogPath = "/usr/share/archie/CHANGELOG.archied.md"
packagedRuntimeChangelogPath = "/usr/share/archie/CHANGELOG.archie.md"
)
@@ -87,15 +95,19 @@ var (
func chatGenerateOptions(
messages []chat.Message,
registry *tools.Registry,
maxSteps int,
) (core.GenerateOptions, error) {
toolSet, err := agentexec.BuildToolSet(registry)
if err != nil {
return core.GenerateOptions{}, err
}
if maxSteps <= 0 {
maxSteps = defaultChatMaxSteps
}
return core.GenerateOptions{
Messages: messages,
Tools: toolSet,
MaxSteps: defaultChatMaxSteps,
MaxSteps: maxSteps,
}, nil
}
+15 -2
View File
@@ -38,7 +38,7 @@ func TestChatGenerateOptionsIncludesToolsAndMultipleSteps(t *testing.T) {
}
messages := []chat.Message{{Role: chat.RoleUser, Content: "remember this"}}
options, err := chatGenerateOptions(messages, registry)
options, err := chatGenerateOptions(messages, registry, 0)
if err != nil {
t.Fatal(err)
}
@@ -64,11 +64,24 @@ func TestChatGenerateOptionsReportsInvalidToolSchemas(t *testing.T) {
}); err != nil {
t.Fatal(err)
}
if _, err := chatGenerateOptions(nil, registry); err == nil {
if _, err := chatGenerateOptions(nil, registry, 0); err == nil {
t.Fatal("chatGenerateOptions() error = nil, want invalid tool schema error")
}
}
// TestChatGenerateOptionsHonoursConfiguredMaxSteps keeps the step budget
// deployment-controlled. A cap that cannot be raised without a rebuild is
// the situation this replaced.
func TestChatGenerateOptionsHonoursConfiguredMaxSteps(t *testing.T) {
options, err := chatGenerateOptions(nil, tools.NewRegistry(), 250)
if err != nil {
t.Fatal(err)
}
if options.MaxSteps != 250 {
t.Errorf("MaxSteps = %d, want the configured 250", options.MaxSteps)
}
}
func TestGatewaysStartOnlyAfterCapabilityHostAndDaemonStartup(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
+1 -1
View File
@@ -180,7 +180,7 @@ func makeTelegramLLMResponder(ctx context.Context, tg *telegram.Gateway, s teleg
compressed = append(compressed, gateway.CompressedMessage{Role: role, Content: h.Text})
}
view := gateway.CompressHistory(compressed, gateway.DefaultCompressionConfig())
options, err := chatGenerateOptions(nil, s.ToolReg)
options, err := chatGenerateOptions(nil, s.ToolReg, s.Cfg.Chat.MaxSteps)
if err != nil {
return "", fmt.Errorf("build chat tools: %w", err)
}
+8
View File
@@ -55,9 +55,17 @@ class = "ollama"
# shell, grep, find). Unset, those tools are not registered at all. Set it and
# the agent can read, modify and execute inside that directory with the same
# reach the daemon process has -- choose it deliberately.
#
# `max_steps` caps the model/tool round-trips in a single chat turn
# (default 100). This is a task budget, not a safety limit: one question
# about a codebase easily costs a read, several greps and a few edits, and
# a turn that runs out stops mid-task looking like it gave up. To interrupt
# a turn, send /stop -- that cancels it immediately and clears anything
# queued behind it.
# [chat]
# operator = "Ada Lovelace"
# workspace = "/var/lib/archie/workspace"
# max_steps = 100
# models = [
# "openai/gpt-5.6-sol",
# "openrouter/openai/gpt-5.6-sol",
+62 -11
View File
@@ -12,6 +12,8 @@ import (
"github.com/go-telegram/bot"
"github.com/go-telegram/bot/models"
"github.com/samcharles93/archie-core/internal/gateway"
)
// DangerousCommandAuthority is the sandbox/process/filesystem authority
@@ -405,25 +407,74 @@ func rollbackApprovalText(num int) string {
// ── /stop handler ──────────────────────────────────────────────
// stopHandler serves /stop: terminate a background process after approval.
func (g *Gateway) stopHandler() bot.HandlerFunc {
// stopCurrentTurn cancels whatever this conversation is currently doing.
//
// Cancellation propagates through the turn's context, so it reaches the
// model stream and any tool running underneath it -- a shell command built
// with exec.CommandContext is killed with the turn, which is the case that
// matters. Whatever the reply had already streamed stays on screen: it is
// the record of what Archie did before being stopped.
func (g *Gateway) stopCurrentTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) {
var cancelled bool
var dropped int
// turns is only built by launch, so a gateway that has never started
// simply has nothing to stop. Report that rather than failing.
if g.turns != nil && router != nil {
session, err := router.ResolveSessionKey(ctx, gateway.Message{
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
ThreadID: threadIDString(msg.MessageThreadID),
From: msg.From.Username,
Text: msg.Text,
})
if err != nil {
g.log.Error("resolve session for stop", "error", err)
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.")
return
}
cancelled, dropped = g.turns.Stop(session)
g.log.Info("stop requested", "session", session, "cancelled", cancelled, "dropped", dropped)
}
switch {
case cancelled && dropped > 0:
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
fmt.Sprintf("🛑 Stopped. %d queued message(s) discarded.", dropped))
case cancelled:
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "🛑 Stopped.")
case dropped > 0:
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
fmt.Sprintf("🛑 Nothing was running. %d queued message(s) discarded.", dropped))
default:
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
"Nothing is running.\n\nUse /stop <process-name> to terminate a background process.")
}
}
// stopHandler serves /stop.
//
// Bare /stop is the emergency brake: it cancels the conversation's running
// turn immediately, with no approval step. Approval exists to protect
// against destructive commands, and stopping is the opposite -- gating it
// behind a confirmation round-trip would defeat the entire point.
//
// /stop <process-name> keeps the original behaviour of terminating a named
// background process, which is destructive and still requires approval.
func (g *Gateway) stopHandler(router *gateway.Router) bot.HandlerFunc {
return func(ctx context.Context, b *bot.Bot, update *models.Update) {
msg, ok := g.authorizedMessage(ctx, b, update)
if !ok {
return
}
if g.Dangerous == nil {
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.")
rest := strings.TrimSpace(restAfterTelegram(msg.Text, "/stop", ""))
if rest == "" {
g.stopCurrentTurn(ctx, b, msg, router)
return
}
rest := restAfterTelegram(msg.Text, "/stop", "")
rest = strings.TrimSpace(rest)
if rest == "" {
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID,
"Usage: /stop <process-name>\n\nTerminate a running background process. "+
"This requires approval because it may discard in-progress work.")
if g.Dangerous == nil {
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "Process control is not configured for this installation.")
return
}
+56 -7
View File
@@ -11,6 +11,8 @@ import (
"time"
"github.com/go-telegram/bot/models"
"github.com/samcharles93/archie-core/internal/gateway"
)
// ── DangerousCommandAuthority stub ─────────────────────────────
@@ -303,7 +305,7 @@ func TestStopRequiresApproval(t *testing.T) {
g.Dangerous = stub
b, requests := newTelegramTestBot(t)
g.stopHandler()(context.Background(), b, &models.Update{
g.stopHandler(nil)(context.Background(), b, &models.Update{
Message: &models.Message{
From: &models.User{ID: allowedUserID},
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
@@ -333,7 +335,7 @@ func TestStopApprovalExecutes(t *testing.T) {
b, requests := newTelegramTestBot(t)
// Step 1: Trigger the approval prompt.
g.stopHandler()(context.Background(), b, &models.Update{
g.stopHandler(nil)(context.Background(), b, &models.Update{
Message: &models.Message{
From: &models.User{ID: allowedUserID},
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
@@ -367,14 +369,18 @@ func TestStopApprovalExecutes(t *testing.T) {
}
}
func TestStopNoArgument(t *testing.T) {
// TestStopWithNoArgumentDoesNotRequireApproval is the emergency-brake
// contract. Bare /stop targets the running chat turn, so it must act
// immediately -- if it ever renders an approval prompt again, the brake has
// been put behind a confirmation round-trip and no longer works.
func TestStopWithNoArgumentDoesNotRequireApproval(t *testing.T) {
const allowedUserID = int64(42)
stub := &dangerousStub{}
g := New("1:test", "", "", []int64{allowedUserID}, slog.Default())
g.Dangerous = stub
b, requests := newTelegramTestBot(t)
g.stopHandler()(context.Background(), b, &models.Update{
g.stopHandler(nil)(context.Background(), b, &models.Update{
Message: &models.Message{
From: &models.User{ID: allowedUserID},
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
@@ -383,8 +389,51 @@ func TestStopNoArgument(t *testing.T) {
})
text := messageText(*requests, 0)
if !strings.Contains(text, "Usage: /stop") {
t.Errorf("expected usage message, got: %s", text)
if strings.Contains(text, "Dangerous command") {
t.Errorf("bare /stop asked for approval: %s", text)
}
if !strings.Contains(text, "Nothing is running") {
t.Errorf("expected an idle report, got: %s", text)
}
if len(stub.stopCalls) != 0 {
t.Errorf("bare /stop terminated a named process: %v", stub.stopCalls)
}
}
// TestStopCancelsRunningChatTurn covers the whole point of the command:
// a turn in flight is cancelled, and the queue behind it is discarded.
func TestStopCancelsRunningChatTurn(t *testing.T) {
const allowedUserID = int64(42)
g := New("1:test", "", "", []int64{allowedUserID}, slog.Default())
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
g.turns = gateway.NewTurns(slog.Default())
// stopCurrentTurn resolves the session through the router; with none
// wired, drive the lane directly and assert on the lane's state.
const session = "chat:7"
started := make(chan struct{})
stopped := make(chan struct{})
g.turns.Submit(ctx, session, func(turnCtx context.Context) {
close(started)
<-turnCtx.Done()
close(stopped)
})
<-started
g.turns.Submit(ctx, session, func(context.Context) {})
cancelled, dropped := g.turns.Stop(session)
if !cancelled {
t.Error("the running turn was not cancelled")
}
if dropped != 1 {
t.Errorf("dropped = %d, want 1", dropped)
}
select {
case <-stopped:
case <-time.After(2 * time.Second):
t.Fatal("the turn did not observe cancellation")
}
}
@@ -851,7 +900,7 @@ func TestStopNotConfigured(t *testing.T) {
// Dangerous is nil.
b, requests := newTelegramTestBot(t)
g.stopHandler()(context.Background(), b, &models.Update{
g.stopHandler(nil)(context.Background(), b, &models.Update{
Message: &models.Message{
From: &models.User{ID: allowedUserID},
Chat: models.Chat{ID: 7, Type: models.ChatTypePrivate},
+56 -11
View File
@@ -6,6 +6,7 @@ package telegram
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
@@ -71,6 +72,11 @@ type Gateway struct {
dangerousActions map[string]dangerousAction
permanentApprovals []permanentApproval
// turns serialises chat turns per session off the update worker and
// makes the running one cancellable by /stop. Rebuilt on every launch
// so a restart abandons in-flight turns with the old bot instance.
turns *gateway.Turns
log *slog.Logger
bot *bot.Bot
webhookCancel context.CancelFunc
@@ -162,6 +168,11 @@ func (g *Gateway) Start(ctx context.Context, router *gateway.Router) error {
// launch builds one bot instance, registers handlers and begins receiving.
// It returns as soon as delivery is running; the caller owns ctx.
func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot, error) {
// Turns are per-launch. Each queued turn carries the update handler's
// context, which is this launch's, so a /restart cancels everything
// still running under the outgoing bot instance.
g.turns = gateway.NewTurns(g.log)
opts := []bot.Option{
bot.WithErrorsHandler(func(err error) {
g.log.Error("telegram pipeline error", "error", err)
@@ -195,7 +206,7 @@ func (g *Gateway) launch(ctx context.Context, router *gateway.Router) (*bot.Bot,
b.RegisterHandler(bot.HandlerTypeMessageText, "/help", bot.MatchTypeExact, g.helpHandler())
b.RegisterHandler(bot.HandlerTypeMessageText, "/restart", bot.MatchTypeExact, g.restartHandler())
b.RegisterHandler(bot.HandlerTypeMessageText, "/rollback", bot.MatchTypePrefix, g.rollbackHandler())
b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler())
b.RegisterHandler(bot.HandlerTypeMessageText, "/stop", bot.MatchTypePrefix, g.stopHandler(router))
b.RegisterHandler(bot.HandlerTypeMessageText, "/approve", bot.MatchTypeExact, g.approveHandler())
b.RegisterHandler(bot.HandlerTypeMessageText, "/deny", bot.MatchTypeExact, g.denyHandler())
@@ -434,29 +445,63 @@ func (g *Gateway) defaultHandler(router *gateway.Router) bot.HandlerFunc {
g.handlePersonalityCommand(ctx, b, msg, router)
return
}
g.submitTurn(ctx, b, msg, router)
}
}
// submitTurn hands the message to its session's turn lane and returns at
// once, leaving the bot's single update worker free.
//
// The turn must not run on the caller's goroutine. go-telegram/bot serves
// updates from one worker that invokes handlers synchronously, so a turn
// running here would block delivery of every later update -- including the
// /stop meant to cancel it, which would sit unread until the turn it was
// aimed at had already finished.
func (g *Gateway) submitTurn(ctx context.Context, b *bot.Bot, msg *models.Message, router *gateway.Router) {
gm := gateway.Message{
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
ThreadID: threadIDString(msg.MessageThreadID),
From: msg.From.Username,
Text: msg.Text,
}
// The lane key must be the session, so that /stop -- which resolves
// the same key -- reaches the turn the sender is actually watching.
session, err := router.ResolveSessionKey(ctx, gm)
if err != nil {
g.log.Error("resolve session for turn", "error", err)
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, "❌ Could not resolve this conversation's session.")
return
}
chatID, threadID := msg.Chat.ID, msg.MessageThreadID
g.turns.Submit(ctx, session, func(turnCtx context.Context) {
// reply appears as it is written; the typing indicator covers the
// gap before the first token and any non-streaming path.
stopTyping := g.startTyping(ctx, b, msg.Chat.ID, msg.MessageThreadID)
draft := g.newDraft(b, msg.Chat.ID, msg.MessageThreadID)
stopTyping := g.startTyping(turnCtx, b, chatID, threadID)
draft := g.newDraft(b, chatID, threadID)
// If it starts with / but wasn't matched by a registered
// handler, it's unknown -- let the router handle it (which
// will say "unrecognized").
reply, err := router.RouteStream(ctx, gateway.Message{
ChannelID: fmt.Sprintf("%d", msg.Chat.ID),
ThreadID: threadIDString(msg.MessageThreadID),
From: msg.From.Username,
Text: msg.Text,
}, draft.onDelta)
reply, err := router.RouteStream(turnCtx, gm, draft.onDelta)
stopTyping()
// A cancelled turn is a /stop, not a fault. The stop handler has
// already acknowledged it, and turnCtx is dead, so there is
// nothing useful left to send from here.
if errors.Is(err, context.Canceled) {
g.log.Info("chat turn stopped", "session", session)
return
}
if err != nil {
g.log.Error("route failed", "error", err)
return
}
if reply != "" {
g.sendMessage(ctx, b, msg.Chat.ID, msg.MessageThreadID, reply)
g.sendMessage(turnCtx, b, chatID, threadID, reply)
}
}
})
}
// ── helpers ──────────────────────────────────────────────────
+9
View File
@@ -491,6 +491,15 @@ type ChatConfig struct {
// they read, write and execute, so the directory must be a deliberate
// choice rather than whatever the daemon happens to start in.
Workspace string `toml:"workspace" yaml:"workspace"`
// MaxSteps caps how many model/tool round-trips one chat turn may take
// before the runtime stops and returns what it has. Zero uses the
// default.
//
// This is a real task budget, not a safety limit: a single question
// about a codebase routinely costs a read, several greps and a handful
// of edits, and running out mid-turn looks to the user like the agent
// simply gave up. Use /stop to interrupt, not a small step cap.
MaxSteps int `toml:"max_steps" yaml:"max_steps"`
// Models is the optional interactive-chat model catalog. When empty,
// chat falls back to the distinct model references assigned to workflow
// roles in the top-level [models] table.
+229
View File
@@ -0,0 +1,229 @@
package gateway
import (
"context"
"log/slog"
"runtime/debug"
"sync"
)
// Turns serialises chat turns per session and keeps the running turn
// cancellable.
//
// Channel event loops are typically single-threaded -- go-telegram/bot,
// for one, runs a single worker that calls handlers synchronously. Running
// a turn on that goroutine blocks delivery of every subsequent update,
// including the command meant to stop it: /stop would sit unread in the
// update channel until the turn it was meant to kill had already finished.
//
// Submitting to Turns hands the work to a per-session goroutine and returns
// immediately, so the event loop stays free and control commands are always
// serviceable. Ordering is preserved by the lane being serial, not by the
// event loop being blocked.
//
// Queues are unbounded and never shed. A dropped message is a silent
// failure that surfaces much later as an agent that ignored something,
// which costs far more to diagnose than the memory a backlog occupies.
// [Turns.Stop] is the way a backlog is cleared -- deliberately, by the
// person who created it.
type Turns struct {
log *slog.Logger
mu sync.Mutex
lanes map[string]*lane
}
// queued is one pending turn: the work, already bound to its own
// cancellable context, and the cancel that releases it.
//
// The context is derived when the turn is submitted rather than when it
// starts running, which means a turn has a live cancel handle for its whole
// time in the backlog. Stop can therefore cancel queued turns rather than
// merely dropping them, and each turn stays tied to the caller that asked
// for it -- it dies when that caller's context does.
type queued struct {
run func()
cancel context.CancelFunc
}
// lane is one session's serial worker.
//
// A single mutex guards the backlog, the running turn's cancel, and whether
// a worker is live, so that Stop observes them together. Splitting them
// would leave a window where a turn finishes between the backlog being
// cleared and the cancel being called, letting a queued turn start after a
// stop.
type lane struct {
mu sync.Mutex
pending []queued
// cancel stops the turn currently running in this lane. It is nil
// whenever the lane is idle.
cancel context.CancelFunc
// working reports whether a worker goroutine is draining this lane.
// The worker exits when the backlog empties, so this is what tells
// Submit whether to start a new one.
working bool
}
// NewTurns returns an idle dispatcher.
func NewTurns(log *slog.Logger) *Turns {
return &Turns{log: log, lanes: make(map[string]*lane)}
}
// Submit schedules run for the given session and returns without waiting
// for it. run receives a context derived from ctx that [Turns.Stop] can
// cancel.
//
// Submit never blocks and never rejects: messages queue.
func (t *Turns) Submit(ctx context.Context, session string, run func(context.Context)) {
t.mu.Lock()
l, ok := t.lanes[session]
if !ok {
l = &lane{}
t.lanes[session] = l
}
t.mu.Unlock()
turnCtx, cancel := context.WithCancel(ctx)
l.mu.Lock()
l.pending = append(l.pending, queued{run: func() { run(turnCtx) }, cancel: cancel})
depth := len(l.pending)
start := !l.working
if start {
l.working = true
}
l.mu.Unlock()
if start {
go l.serve(t.log.With("session", session))
}
if depth > 1 {
t.log.Info("chat turn queued", "session", session, "queue_depth", depth)
}
}
// Stop cancels the session's running turn and discards everything queued
// behind it. It reports whether a turn was actually cancelled and how many
// queued turns were dropped.
//
// Dropping the queue is intentional. "Stop" means stop, not "stop this one
// and immediately begin the next thing I said" -- starting the queued turn
// would look to the sender exactly like the stop had failed.
func (t *Turns) Stop(session string) (cancelled bool, dropped int) {
l := t.lane(session)
if l == nil {
return false, 0
}
l.mu.Lock()
defer l.mu.Unlock()
dropped = len(l.pending)
for i, q := range l.pending {
q.cancel()
l.pending[i] = queued{}
}
l.pending = nil
if l.cancel != nil {
l.cancel()
cancelled = true
}
return cancelled, dropped
}
// Running reports whether the session currently has a turn in flight.
func (t *Turns) Running(session string) bool {
l := t.lane(session)
if l == nil {
return false
}
l.mu.Lock()
defer l.mu.Unlock()
return l.cancel != nil
}
// Queued reports how many turns are waiting behind the running one.
func (t *Turns) Queued(session string) int {
l := t.lane(session)
if l == nil {
return 0
}
l.mu.Lock()
defer l.mu.Unlock()
return len(l.pending)
}
// lane returns the session's lane, or nil if it has never been used.
func (t *Turns) lane(session string) *lane {
t.mu.Lock()
defer t.mu.Unlock()
return t.lanes[session]
}
// serve drains the lane's backlog one turn at a time and exits once it is
// empty. Submit starts a fresh worker when more arrives, so an idle session
// costs no goroutine.
func (l *lane) serve(log *slog.Logger) {
for {
next, ok := l.begin()
if !ok {
return
}
l.runOne(next, log)
}
}
// begin pops the oldest queued turn and publishes its cancel in the same
// critical section, reporting false when the backlog is empty -- at which
// point it also retires the worker.
//
// The pop and the publish must happen together. Were the turn popped first
// and its cancel published after, a Stop landing in between would see an
// empty backlog and no running turn, report that nothing was happening, and
// let the turn it should have killed start regardless.
func (l *lane) begin() (queued, bool) {
l.mu.Lock()
defer l.mu.Unlock()
if len(l.pending) == 0 {
// Retire under the same lock Submit uses to decide whether a
// worker is live, so a turn arriving now either starts a new
// worker or is picked up before this one retires -- never
// neither, which would strand the backlog forever.
l.working = false
return queued{}, false
}
next := l.pending[0]
// Clear the vacated slot so a completed turn's closure -- and whatever
// it captured -- does not stay reachable through the backing array.
l.pending[0] = queued{}
l.pending = l.pending[1:]
l.cancel = next.cancel
return next, true
}
// runOne executes one turn, always retracting its cancel afterwards so a
// later Stop cannot cancel an already-finished turn.
//
// Panics are contained here rather than by the channel's own recovery
// middleware: that middleware wraps the event-loop goroutine, and the turn
// no longer runs on it. Without this, one bad turn would take the lane's
// goroutine down and the session would silently stop answering forever.
func (l *lane) runOne(next queued, log *slog.Logger) {
defer next.cancel()
defer func() {
l.mu.Lock()
l.cancel = nil
l.mu.Unlock()
if r := recover(); r != nil {
log.Error("chat turn panicked", "panic", r, "stack", string(debug.Stack()))
}
}()
next.run()
}
+315
View File
@@ -0,0 +1,315 @@
package gateway_test
import (
"context"
"errors"
"io"
"log/slog"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/samcharles93/archie-core/internal/gateway"
)
func testTurns(t *testing.T) *gateway.Turns {
t.Helper()
return gateway.NewTurns(slog.New(slog.NewTextHandler(io.Discard, nil)))
}
// TestSubmitDoesNotBlockCaller is the whole reason this type exists: the
// channel event loop must stay free while a turn runs, or the command that
// stops the turn can never be delivered.
func TestSubmitDoesNotBlockCaller(t *testing.T) {
t.Parallel()
turns := testTurns(t)
release := make(chan struct{})
started := make(chan struct{})
turns.Submit(context.Background(), "s1", func(context.Context) {
close(started)
<-release
})
<-started
defer close(release)
returned := make(chan struct{})
go func() {
turns.Submit(context.Background(), "s1", func(context.Context) {})
close(returned)
}()
select {
case <-returned:
case <-time.After(2 * time.Second):
t.Fatal("Submit blocked while a turn was running; the event loop would be stuck")
}
}
func TestStopCancelsRunningTurn(t *testing.T) {
t.Parallel()
turns := testTurns(t)
started := make(chan struct{})
var cause atomic.Value
turns.Submit(context.Background(), "s1", func(ctx context.Context) {
close(started)
<-ctx.Done()
cause.Store(ctx.Err())
})
<-started
cancelled, dropped := turns.Stop("s1")
if !cancelled {
t.Error("Stop reported no running turn")
}
if dropped != 0 {
t.Errorf("dropped = %d, want 0", dropped)
}
waitFor(t, func() bool { return cause.Load() != nil })
if err, _ := cause.Load().(error); !errors.Is(err, context.Canceled) {
t.Errorf("turn context error = %v, want context.Canceled", err)
}
}
// TestStopDropsQueuedTurns guards the "stop means stop" rule. If the
// queued turn ran after cancellation, the sender would see a new reply
// begin immediately and conclude the stop had failed.
func TestStopDropsQueuedTurns(t *testing.T) {
t.Parallel()
turns := testTurns(t)
started := make(chan struct{})
release := make(chan struct{})
var queuedRan atomic.Bool
turns.Submit(context.Background(), "s1", func(context.Context) {
close(started)
<-release
})
<-started
for range 2 {
turns.Submit(context.Background(), "s1", func(context.Context) { queuedRan.Store(true) })
}
cancelled, dropped := turns.Stop("s1")
if !cancelled || dropped != 2 {
t.Fatalf("Stop() = (%v, %d), want (true, 2)", cancelled, dropped)
}
close(release)
// Give the lane time to pull anything that survived the drain.
time.Sleep(100 * time.Millisecond)
if queuedRan.Load() {
t.Error("a queued turn ran after /stop")
}
}
// TestStopLeavesTheCallersContextAlone checks that Stop only reaches the
// contexts Turns derived. Callers pass in the channel's long-lived run
// context, so cancelling that instead of the turn's own would take the
// whole gateway down every time someone sent /stop.
func TestStopLeavesTheCallersContextAlone(t *testing.T) {
t.Parallel()
turns := testTurns(t)
callerCtx, callerCancel := context.WithCancel(context.Background())
defer callerCancel()
started := make(chan struct{})
turns.Submit(callerCtx, "s1", func(ctx context.Context) {
close(started)
<-ctx.Done()
})
<-started
turns.Submit(callerCtx, "s1", func(context.Context) {})
if cancelled, dropped := turns.Stop("s1"); !cancelled || dropped != 1 {
t.Fatalf("Stop() = (%v, %d), want (true, 1)", cancelled, dropped)
}
if err := callerCtx.Err(); err != nil {
t.Errorf("Stop cancelled the caller's context: %v", err)
}
// The lane must still be usable afterwards.
ran := make(chan struct{})
turns.Submit(callerCtx, "s1", func(context.Context) { close(ran) })
select {
case <-ran:
case <-time.After(2 * time.Second):
t.Fatal("the session stopped accepting turns after /stop")
}
}
// TestDeepBacklogIsKeptAndOrdered is the no-shedding contract. A message
// that is silently dropped surfaces much later as an agent that ignored
// something, which is far more expensive to diagnose than holding the
// backlog. Every message queues, and they run in the order they arrived.
func TestDeepBacklogIsKeptAndOrdered(t *testing.T) {
t.Parallel()
const backlog = 100
turns := testTurns(t)
started := make(chan struct{})
release := make(chan struct{})
turns.Submit(context.Background(), "s1", func(context.Context) {
close(started)
<-release
})
<-started
var mu sync.Mutex
var order []int
done := make(chan struct{}, backlog)
for i := range backlog {
turns.Submit(context.Background(), "s1", func(context.Context) {
mu.Lock()
order = append(order, i)
mu.Unlock()
done <- struct{}{}
})
}
if got := turns.Queued("s1"); got != backlog {
t.Fatalf("Queued() = %d, want all %d held", got, backlog)
}
close(release)
for range backlog {
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("queued turns did not all run")
}
}
mu.Lock()
defer mu.Unlock()
if len(order) != backlog {
t.Fatalf("ran %d turns, want %d", len(order), backlog)
}
for i, got := range order {
if got != i {
t.Fatalf("turn %d ran at position %d; the queue is not FIFO", got, i)
}
}
}
// TestTurnsRunSeriallyWithinSession covers the queueing half of the
// contract: replies must not interleave for one conversation.
func TestTurnsRunSeriallyWithinSession(t *testing.T) {
t.Parallel()
turns := testTurns(t)
var mu sync.Mutex
var concurrent, peak int
done := make(chan struct{}, 3)
for range 3 {
turns.Submit(context.Background(), "s1", func(context.Context) {
mu.Lock()
concurrent++
if concurrent > peak {
peak = concurrent
}
mu.Unlock()
time.Sleep(20 * time.Millisecond)
mu.Lock()
concurrent--
mu.Unlock()
done <- struct{}{}
})
}
for range 3 {
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("turns did not complete")
}
}
if peak != 1 {
t.Errorf("peak concurrency = %d, want 1; turns in one session overlapped", peak)
}
}
// TestSessionsAreIndependent checks that one busy session cannot stall
// another, and that Stop is scoped to the session that asked for it.
func TestSessionsAreIndependent(t *testing.T) {
t.Parallel()
turns := testTurns(t)
busy := make(chan struct{})
release := make(chan struct{})
defer close(release)
turns.Submit(context.Background(), "busy", func(context.Context) {
close(busy)
<-release
})
<-busy
ran := make(chan struct{})
turns.Submit(context.Background(), "other", func(context.Context) { close(ran) })
select {
case <-ran:
case <-time.After(2 * time.Second):
t.Fatal("a second session was blocked behind the first")
}
if cancelled, _ := turns.Stop("other"); cancelled {
t.Error("Stop cancelled a turn in an idle session")
}
}
// TestStopAfterCompletionIsInert guards the retraction in runOne: a stale
// cancel left behind by a finished turn would kill the next one.
func TestStopAfterCompletionIsInert(t *testing.T) {
t.Parallel()
turns := testTurns(t)
done := make(chan struct{})
turns.Submit(context.Background(), "s1", func(context.Context) { close(done) })
<-done
waitFor(t, func() bool { return !turns.Running("s1") })
if cancelled, dropped := turns.Stop("s1"); cancelled || dropped != 0 {
t.Errorf("Stop() = (%v, %d) on an idle session, want (false, 0)", cancelled, dropped)
}
}
// TestPanicInTurnDoesNotKillLane covers the recovery in runOne. Losing the
// lane goroutine would leave the session permanently unable to reply.
func TestPanicInTurnDoesNotKillLane(t *testing.T) {
t.Parallel()
turns := testTurns(t)
turns.Submit(context.Background(), "s1", func(context.Context) { panic("boom") })
ran := make(chan struct{})
turns.Submit(context.Background(), "s1", func(context.Context) { close(ran) })
select {
case <-ran:
case <-time.After(2 * time.Second):
t.Fatal("the lane died with the panicking turn")
}
}
func waitFor(t *testing.T, cond func() bool) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if cond() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("condition not met within timeout")
}