diff --git a/cmd/archied/main.go b/cmd/archied/main.go index bdbbe8c28..2e2cb8290 100644 --- a/cmd/archied/main.go +++ b/cmd/archied/main.go @@ -354,6 +354,10 @@ func run() int { // problem into a crash loop with the real error scrolling past unread. containerPool, storeBackend, closeDocker := startContainers(ctx, cfg, log) defer closeDocker() + // Sandboxing that was asked for and could not start must not silently + // become no sandboxing: the daemon stays up, but tasks park rather than + // running the agent loop on the host. + sandboxRequired := cfg.Containers.Enabled && containerPool == nil // ── LLM runtime ────────────────────────────────────────────────── // Created before gateways so the LLMResponder can be wired into the @@ -667,23 +671,24 @@ func run() int { // via Yaegi is deferred to the container/agent runtime. d := &daemon.Daemon{ - Cfg: cfg, - Store: st, - Bus: bus, - Forge: forgeClient, - Trees: trees, - Agent: agentRunner, - Workflows: registry, - CapabilityHost: capabilityHost, - Storage: storeBackend, - Log: log, - CustomStages: wfeval.Discover, - Tasks: natsClient, - ContainerPool: containerPool, - Memory: memManager, - Guardrails: guardrails, - ToolRegistry: toolReg, - Identities: identityRunners, + Cfg: cfg, + Store: st, + Bus: bus, + Forge: forgeClient, + Trees: trees, + Agent: agentRunner, + Workflows: registry, + CapabilityHost: capabilityHost, + Storage: storeBackend, + Log: log, + CustomStages: wfeval.Discover, + Tasks: natsClient, + ContainerPool: containerPool, + SandboxRequired: sandboxRequired, + Memory: memManager, + Guardrails: guardrails, + ToolRegistry: toolReg, + Identities: identityRunners, } // Give /cancel and /stop a handle on work already in flight. The diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 39c4a5419..a1eac41a7 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -69,6 +69,14 @@ type Daemon struct { // TaskRunRetryBackoff is the delay between retry attempts within // TaskRunReadyTimeout. Zero uses defaultTaskRunRetryBackoff. TaskRunRetryBackoff time.Duration + // SandboxRequired reports that [containers] asked for sandboxing that + // could not be brought up. It exists because ContainerPool == nil means + // two very different things: "sandboxing was never configured, run + // in-process" and "sandboxing was configured and failed". Treating them + // alike silently dropped the isolation boundary and ran agent loops on + // the host with the daemon's own authority. + SandboxRequired bool + // ContainerPool manages Docker container lifecycle. Nil when [containers] // is not configured. When non-nil, every task gets a fresh container. ContainerPool *container.Pool @@ -797,6 +805,24 @@ func (d *Daemon) process(ctx context.Context, task *store.Task) { task.Branch = branch _ = d.Store.Update(ctx, task) + // Refuse to run unsandboxed when sandboxing was asked for. Parking is + // recoverable -- fix the image, retry -- whereas running the agent loop + // in-process would hand it the host filesystem and the daemon's own + // authority, which is the one thing [containers] exists to prevent. The + // daemon itself stays up: chat and the dashboard are unaffected. + if d.SandboxRequired && d.ContainerPool == nil { + const reason = "containers are enabled but the pool is unavailable; " + + "refusing to run this task unsandboxed on the host" + d.Log.Error("task parked: sandbox unavailable", "task", task.ID, + "hint", "run `docker compose pull agent` (or `build agent`), then retry the task") + _ = d.Store.Transition(ctx, task.ID, store.StatusRunning, store.StatusParked, reason) + d.emit(events.Event{ + Kind: events.KindParked, TaskID: task.ID, + Repo: repo.FullName(), Issue: task.IssueNumber, Detail: reason, + }) + return + } + // Acquire a container for the task when Docker sandboxing is enabled. if d.ContainerPool != nil { ctr, ok := d.acquireTaskContainer(ctx, task, repo, workDir) diff --git a/internal/gateway/compress_ordering_test.go b/internal/gateway/compress_ordering_test.go new file mode 100644 index 000000000..4e012c099 --- /dev/null +++ b/internal/gateway/compress_ordering_test.go @@ -0,0 +1,114 @@ +package gateway + +import ( + "context" + "strings" + "testing" +) + +// The summary stands in for the messages it replaced, so it belongs where +// they were: after the protected head, before the protected tail. +// +// Preserving the retained messages' original timestamps -- which is what +// stops compression destroying their identity -- means ordering is no longer +// the order the caller wrote them in. The summary is a new record, so the +// monotonic clamp stamps it with now and it sorts after everything it was +// supposed to precede. The model then reads the whole tail of the +// conversation and is told afterwards that earlier context was summarised. +func TestCompressSummaryLandsBetweenHeadAndTail(t *testing.T) { + for name, newStore := range compressBackends() { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := newStore(t) + r, sessionID := routerOn(t, store, "chat-1") + + seedForCompress(t, store, sessionID, 40) + if _, err := r.applyCompress(ctx, sessionID, DefaultCompressionConfig()); err != nil { + t.Fatalf("applyCompress: %v", err) + } + + after, err := store.RecentMessages(ctx, sessionID, 1000) + if err != nil { + t.Fatalf("RecentMessages: %v", err) + } + + marker := DefaultCompressionConfig().SummaryMarker + idx := -1 + for i, m := range after { + if strings.Contains(m.Text, marker) { + idx = i + break + } + } + if idx < 0 { + t.Fatal("no summary in the compressed history") + } + + cfg := DefaultCompressionConfig() + if idx != cfg.ProtectFirst { + t.Errorf("summary is at index %d of %d, want %d (after the %d "+ + "protected head messages, before the tail): the model reads "+ + "the recent conversation and is only then told that earlier "+ + "context was summarised", + idx, len(after), cfg.ProtectFirst, cfg.ProtectFirst) + } + if idx == len(after)-1 { + t.Error("the summary is the newest message in the session") + } + }) + } +} + +// PriorReply decides "was this message already answered?" by looking at +// whether the next turn came from the bot. A compression summary is +// attributed to the bot and is not an answer to anything, so it must not be +// mistaken for one -- otherwise a redelivered question is answered with a +// compression banner and the model is never called. +func TestPriorReplyIgnoresACompressionSummary(t *testing.T) { + const ( + sessionID = "chat-1" + identity = "archie" + ) + marker := DefaultCompressionConfig().SummaryMarker + + history := []Message{ + { + MessageID: CanonicalMessageID(sessionID, "tg-unanswered"), + SourceID: "tg-unanswered", + From: "alice", + Text: "what is 2+2?", + }, + { + MessageID: "summary-record", + From: identity, + Text: marker + "\n\n[36 messages removed (~9000 tokens)]", + }, + } + + if got := PriorReply(history, sessionID, "tg-unanswered", identity); got != "" { + t.Errorf("PriorReply = %q, want \"\": the question was never answered, "+ + "so redelivering it must re-run the model rather than replay a "+ + "compression banner as the answer", got) + } +} + +// A genuine reply is still recognised, so the dedup that stops a redelivered +// message being answered twice keeps working. +func TestPriorReplyStillMatchesARealReply(t *testing.T) { + const ( + sessionID = "chat-1" + identity = "archie" + ) + history := []Message{ + { + MessageID: CanonicalMessageID(sessionID, "tg-1"), + SourceID: "tg-1", + From: "alice", + Text: "ping", + }, + {MessageID: "reply", From: identity, Text: "pong"}, + } + if got := PriorReply(history, sessionID, "tg-1", identity); got != "pong" { + t.Errorf("PriorReply = %q, want %q", got, "pong") + } +} diff --git a/internal/gateway/session_commands.go b/internal/gateway/session_commands.go index 6f7a34b23..f9b1eba9a 100644 --- a/internal/gateway/session_commands.go +++ b/internal/gateway/session_commands.go @@ -42,6 +42,7 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel // poisoned entry repairs itself instead of persisting every subsequent // message under session "". if ok && id != "" { + t.touch(ctx, id) return id, nil } @@ -73,7 +74,9 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel } } if best != nil { - return t.cacheActive(key, best.SessionID), nil + resolved := t.cacheActive(key, best.SessionID) + t.touch(ctx, resolved) + return resolved, nil } // Create a new session with a deterministic key so it matches the @@ -111,6 +114,23 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel return id, nil } +// touch records that a session is in use now. +// +// Nothing called Touch before, so LastActiveAt only ever held the creation +// time and "resolve the most recently active session" really meant the most +// recently created one -- an operator who switched to an older session with +// /topic and worked there was dropped back into the newer, abandoned one on +// the next restart. +// +// Best-effort: a failed timestamp update must not fail the turn, and the +// worst case is the ordering this restores being stale again. +func (t *sessionTracker) touch(ctx context.Context, sessionID string) { + if sessionID == "" { + return + } + _ = t.sessions.Touch(ctx, sessionID) +} + // claimActive reserves key for sessionID unless another caller already holds // it. It returns the winning session ID and whether this caller won. // @@ -472,6 +492,14 @@ func (r *Router) handleBranch(ctx context.Context, msg Message, rest string) (st channelID := msg.ChannelID threadID := msg.ThreadID parentID := r.sessionTracker.getActive(channelID, threadID) + if parentID == "" { + // Nothing to branch from. Telegram never reaches this -- submitTurn + // resolves first -- but the email and webhook channels call Route + // directly, and slicing an empty parent ID for the default title + // panicked. The email path runs in a goroutine with no recover, so + // that took the daemon down over a single message. + return "No active session to branch from. Send a message first.", nil + } branchName := strings.TrimSpace(rest) @@ -657,16 +685,45 @@ func compressedHistory(original []Message, view CompressedView, identity string) head := min(view.ProtectedFirst, len(original)) tail := min(view.ProtectedLast, len(original)-head) + // The summary stands in for the span it replaced, so it takes a + // timestamp inside that span. Ordering is by timestamp, not by the order + // the caller writes: the retained messages keep their original times -- + // that is what preserves their identity -- so a summary stamped with now + // would sort after the tail it is supposed to precede, and the model + // would read the recent conversation before being told earlier context + // was summarised. + if summaryAt >= len(view.Messages) { + // A view whose summary index is outside its own messages cannot be + // reconstructed; leaving history untouched is the safe answer. + return original + } out := make([]Message, 0, head+1+tail) out = append(out, original[:head]...) out = append(out, Message{ From: identity, Text: view.Messages[summaryAt].Content, + At: summaryInstant(original, head, tail), }) out = append(out, original[len(original)-tail:]...) return out } +// summaryInstant returns a time inside the gap the summary fills: after the +// last retained head message and before the first retained tail message. +func summaryInstant(original []Message, head, tail int) time.Time { + // The first message the summary replaces is the natural anchor -- the + // summary begins where the summarised span began. + if head < len(original)-tail { + return original[head].At + } + // Nothing was actually replaced. Sit just after the head instead of + // colliding with the tail. + if head > 0 { + return original[head-1].At.Add(time.Millisecond) + } + return time.Time{} +} + func (r *Router) messagesToCompressed(msgs []Message) []CompressedMessage { compressed := make([]CompressedMessage, 0, len(msgs)) for _, h := range msgs { diff --git a/internal/gateway/session_resolve_test.go b/internal/gateway/session_resolve_test.go index 9d61bf9ad..77d03b44f 100644 --- a/internal/gateway/session_resolve_test.go +++ b/internal/gateway/session_resolve_test.go @@ -193,3 +193,101 @@ func sessionIDs(ss []SessionContext) []string { } return out } + +// The two backends must order the same records identically. The Go side used +// to prefer LastActiveAt whenever it was set, while SQLite used +// MAX(last_active_at, created_at); those agree only when LastActiveAt is the +// later of the two. A clock step, or a record written by an older version, +// made the store's own ORDER BY and the caller's pick disagree. +func TestRecencyAgreesWhenLastActivePredatesCreation(t *testing.T) { + for name, newStore := range compressBackends() { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := newStore(t) + + // X was created later but carries an older LastActiveAt. + save := func(id string, created, active time.Duration) { + t.Helper() + if err := store.Save(ctx, SessionContext{ + SessionID: id, + Source: SessionSource{Platform: "telegram", BotUser: "archie", ChannelID: "chat-1"}, + CreatedAt: base.Add(created), + LastActiveAt: base.Add(active), + }); err != nil { + t.Fatalf("Save %s: %v", id, err) + } + } + save("x-created-late", 5*time.Hour, -5*time.Hour) + save("y-normal", 3*time.Hour, 3*time.Hour) + + got, err := store.GetByChannel(ctx, "telegram", "chat-1") + if err != nil { + t.Fatalf("GetByChannel: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d sessions, want 2", len(got)) + } + if got[0].SessionID != "x-created-late" { + t.Errorf("order = %v, want x-created-late first (its creation is "+ + "the later instant)", sessionIDs(got)) + } + + // The caller's own pick must match the store's order. + tr := newSessionTracker(store) + resolved, err := tr.resolve(ctx, "telegram", "archie", "chat-1", "") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if resolved != got[0].SessionID { + t.Errorf("resolve = %q but the store orders %q first: the caller "+ + "and the store disagree", resolved, got[0].SessionID) + } + }) + } +} + +// Resolving a session is what "the session is in use" means, so it must +// update LastActiveAt. Without it the recency ordering ranked by creation +// time, and an operator who switched to an older session with /topic and +// worked there was dropped back into the newer, abandoned one on restart. +func TestResolveMarksTheSessionActive(t *testing.T) { + for name, newStore := range compressBackends() { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := newStore(t) + + // "old" is the one the operator is actually working in. + for _, s := range []struct { + id string + offset time.Duration + }{{"old", 0}, {"new", 5 * time.Hour}} { + if err := store.Save(ctx, SessionContext{ + SessionID: s.id, + Source: SessionSource{Platform: "telegram", BotUser: "archie", ChannelID: "chat-1"}, + CreatedAt: base.Add(s.offset), + LastActiveAt: base.Add(s.offset), + }); err != nil { + t.Fatal(err) + } + } + + tr := newSessionTracker(store) + tr.setActive("chat-1", "", "old") + if got, err := tr.resolve(ctx, "telegram", "archie", "chat-1", ""); err != nil || got != "old" { + t.Fatalf("resolve = (%q, %v), want old", got, err) + } + + // A cold tracker, as after a restart, must now pick "old". + cold := newSessionTracker(store) + got, err := cold.resolve(ctx, "telegram", "archie", "chat-1", "") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if got != "old" { + t.Errorf("after restart resolve = %q, want %q: working in a session "+ + "did not mark it active, so recency still ranked by creation", + got, "old") + } + }) + } +} diff --git a/internal/gateway/session_store.go b/internal/gateway/session_store.go index 26cebfe71..4154b8c1a 100644 --- a/internal/gateway/session_store.go +++ b/internal/gateway/session_store.go @@ -304,8 +304,14 @@ func (s *nellSessionStore) listAllLocked(ctx context.Context) ([]SessionContext, } // sessionRecency is the instant a session was last meaningfully used. +// +// The later of the two timestamps, matching the SQLite side's +// MAX(last_active_at, created_at). Preferring LastActiveAt whenever it was +// set diverged from that whenever it predated CreatedAt -- a clock step, or +// a record written by an older version -- and the two backends then ordered +// the same sessions differently. func sessionRecency(sc SessionContext) time.Time { - if !sc.LastActiveAt.IsZero() { + if sc.LastActiveAt.After(sc.CreatedAt) { return sc.LastActiveAt } return sc.CreatedAt @@ -596,7 +602,22 @@ func (s *nellSessionStore) SaveMessage(ctx context.Context, sessionID string, ms // saveMessageLocked persists one message and reports the document ID it was // written under. Callers must hold s.mu. +// saveMessageLocked appends a message, clamping its timestamp so history +// stays a strictly increasing sequence. func (s *nellSessionStore) saveMessageLocked(ctx context.Context, sessionID string, msg Message) (string, error) { + return s.saveMessageAt(ctx, sessionID, msg, true) +} + +// saveMessageAt writes a message, optionally without the monotonic clamp. +// +// The clamp belongs to the append path: an inbound message must not sort +// ahead of the one that provoked it. It is wrong for a caller rewriting +// history, which is placing records deliberately -- a compression summary +// stands in for the span it replaced and belongs where that span was, not at +// the end. Clamping it there put the summary after the messages it was meant +// to precede, so the model read the recent conversation and was only then +// told that earlier context had been summarised. +func (s *nellSessionStore) saveMessageAt(ctx context.Context, sessionID string, msg Message, clamp bool) (string, error) { db := s.messagesFor(sessionID) // A message carrying a canonical ID keeps it: re-saving one that was // read back must not mint a new identity, or branch points and related @@ -617,9 +638,15 @@ func (s *nellSessionStore) saveMessageLocked(ctx context.Context, sessionID stri return "", fmt.Errorf("sessionstore: read existing message: %w", err) } - at, err := s.nextTimestamp(ctx, sessionID, db, stamp(msg)) - if err != nil { - return "", err + at := stamp(msg) + if clamp { + var err error + if at, err = s.nextTimestamp(ctx, sessionID, db, at); err != nil { + return "", err + } + } else if ms := at.UnixMilli(); ms > s.lastTS[sessionID] { + // Keep the cache honest so a later append still sorts after this. + s.lastTS[sessionID] = ms } if _, err := db.Put(ctx, messageToDoc(id, msg, at)); err != nil { return "", fmt.Errorf("sessionstore: save message: %w", err) @@ -711,7 +738,9 @@ func (s *nellSessionStore) ReplaceMessages( // ReplaceMessages(sess, history[10:]) -- would rewrite the survivors // and then delete them, emptying the session. for _, msg := range msgs { - id, err := s.saveMessageLocked(ctx, sessionID, msg) + // No clamp: the caller is reconstructing a history and has already + // decided where each record belongs. + id, err := s.saveMessageAt(ctx, sessionID, msg, false) if err != nil { return fmt.Errorf("sessionstore: replace messages: %w", err) } @@ -911,10 +940,23 @@ func PriorReply(history []Message, sessionID, sourceID, identity string) string if m.MessageID != id { continue } - if i+1 < len(history) && history[i+1].From == identity { + if i+1 < len(history) && history[i+1].From == identity && + !isCompressionSummary(history[i+1].Text) { return history[i+1].Text } return "" } return "" } + +// isCompressionSummary reports whether a record is a compression summary +// rather than a turn in the conversation. +// +// The summary is attributed to the bot so it replays to the model with the +// right role, which makes it indistinguishable from a reply by sender alone. +// Treating it as one meant a redelivered but unanswered question was +// "answered" with the compression banner and the model was never called -- +// the user's question silently swallowed. +func isCompressionSummary(text string) bool { + return strings.HasPrefix(text, DefaultCompressionConfig().SummaryMarker) +} diff --git a/internal/gateway/session_store_sqlite.go b/internal/gateway/session_store_sqlite.go index d76703828..979fe5418 100644 --- a/internal/gateway/session_store_sqlite.go +++ b/internal/gateway/session_store_sqlite.go @@ -301,6 +301,13 @@ func (s *sqliteSessionStore) SaveMessage(ctx context.Context, sessionID string, // untouched: stored messages are immutable, so a redelivered edit is a // no-op rather than an overwrite. func saveMessage(ctx context.Context, ex execer, sessionID string, msg Message) (string, error) { + return saveMessageAt(ctx, ex, sessionID, msg, true) +} + +// saveMessageAt writes a message, optionally without the monotonic clamp. +// See the NellDB counterpart: the clamp belongs to the append path, not to a +// caller rewriting history and placing records deliberately. +func saveMessageAt(ctx context.Context, ex execer, sessionID string, msg Message, clamp bool) (string, error) { id := msg.MessageID if id == "" { id = newMessageID(sessionID, msg.SourceID) @@ -316,9 +323,12 @@ func saveMessage(ctx context.Context, ex execer, sessionID string, msg Message) return "", fmt.Errorf("sessionstore: read existing message: %w", err) } - at, err := nextTimestampSQL(ctx, ex, sessionID, stamp(msg)) - if err != nil { - return "", err + at := stamp(msg) + if clamp { + var err error + if at, err = nextTimestampSQL(ctx, ex, sessionID, at); err != nil { + return "", err + } } _, err = ex.ExecContext(ctx, ` INSERT INTO messages (message_id, session_id, source_id, sender, text, ts) @@ -402,7 +412,8 @@ func (s *sqliteSessionStore) ReplaceMessages( survivors := make(map[string]struct{}, len(msgs)) for _, msg := range msgs { - id, err := saveMessage(ctx, tx, sessionID, msg) + // No clamp: the caller has already decided where each record belongs. + id, err := saveMessageAt(ctx, tx, sessionID, msg, false) if err != nil { return fmt.Errorf("sessionstore: replace messages: %w", err) } diff --git a/internal/gateway/session_topic_test.go b/internal/gateway/session_topic_test.go index c0843187b..f707c923d 100644 --- a/internal/gateway/session_topic_test.go +++ b/internal/gateway/session_topic_test.go @@ -102,3 +102,25 @@ func TestSetActiveRejectsAnEmptySession(t *testing.T) { t.Fatal("resolve returned empty: an empty setActive left a poisoned entry") } } + +// /branch with no active session sliced an empty parent ID for its default +// title and panicked. Telegram resolves a session before routing, so it never +// reached this -- but the email and webhook channels call Route directly, and +// the email path runs in a goroutine with no recover, so one message took the +// daemon down. +func TestBranchWithNoActiveSessionDoesNotPanic(t *testing.T) { + store := NewSessionStoreMemory("test-node") + t.Cleanup(func() { _ = store.Close() }) + + r := NewRouter(nil, nil, "email") + r.InitSessions(store) + r.Identity = "archie" + + reply, err := r.Route(context.Background(), Message{Text: "/branch", ChannelID: "inbox-1"}) + if err != nil { + t.Fatalf("Route(/branch): %v", err) + } + if reply == "" { + t.Fatal("no reply; the user is told nothing") + } +}