fix(gateway): give messages durable identity and a SQLite store
Three defects shared one root cause -- messages had no identity of their own, so the store could not tell a redelivery from a new message: - SaveMessages destroyed history. Writing a batch replaced records that a concurrent write had already stored, and earlier messages vanished. - A redelivered channel message overwrote the stored one, contradicting the append-only sequence in docs/architecture/messaging-and-work-intake .md:112-114. - /compress stranded whatever exceeded one page: deleting 1200 reported 1000 deleted and left 200 behind. Messages now carry an application-generated MessageID, derived from (session, source id) so a redelivery lands on the same record, and stored records are immutable -- a repeat write is a no-op rather than an overwrite. SourceID keeps the channel-native id for correlation without being load-bearing. _ts is written as Unix milliseconds and clamped monotonic per session: Telegram dates are whole seconds, so replies arriving inside the same second would otherwise order arbitrarily. Deletes page with a cursor instead of assuming one page holds everything. Adds a SQLite session store behind the same interface, per docs/architecture/messaging-and-work-intake.md:107. A shared conformance suite runs all 19 groups against both implementations, so the two cannot drift while the NellDB path is still the one wired up.
This commit is contained in:
1 parent
7d7d03191f
commit
0814f7b9b4
12 files changed
+3257
-475
No files matched your search
@@ -20,6 +20,17 @@ import (
|
||||
|
||||
// Message is an inbound message from a gateway connection.
|
||||
type Message struct {
|
||||
// MessageID is the canonical, application-generated identifier for
|
||||
// this message. It is assigned by the message store, stable, and the
|
||||
// handle branch points and related records correlate on. Populated on
|
||||
// read; ignored on write.
|
||||
MessageID string
|
||||
// SourceID is the channel-native identifier (e.g. a Telegram
|
||||
// message_id). It is external correlation metadata, never the
|
||||
// canonical identity: the store derives a stable MessageID from it so
|
||||
// redelivering an update is idempotent rather than duplicating. Empty
|
||||
// for messages with no upstream identity, which are always appended.
|
||||
SourceID string
|
||||
// ChannelID identifies the conversation within the channel (e.g. a
|
||||
// Telegram chat ID). Replies are sent back to this ID.
|
||||
ChannelID string
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -15,6 +16,10 @@ import (
|
||||
// subsequent messages go to that session.
|
||||
type sessionTracker struct {
|
||||
sessions SessionStore
|
||||
// mu guards active. Channel gateways dispatch each inbound update in
|
||||
// its own goroutine, so resolve, setActive and getActive all run
|
||||
// concurrently.
|
||||
mu sync.RWMutex
|
||||
// active maps "channelID:threadID" → sessionID
|
||||
active map[string]string
|
||||
}
|
||||
@@ -30,11 +35,17 @@ func newSessionTracker(sessions SessionStore) *sessionTracker {
|
||||
// ThreadID may be empty for flat chats.
|
||||
func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channelID, threadID string) (string, error) {
|
||||
key := sessionTrackerKey(channelID, threadID)
|
||||
if id, ok := t.active[key]; ok {
|
||||
t.mu.RLock()
|
||||
id, ok := t.active[key]
|
||||
t.mu.RUnlock()
|
||||
if ok {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Look up existing sessions for this channel+thread.
|
||||
// Look up existing sessions for this channel+thread. The lock is not
|
||||
// held across this I/O: concurrent callers may duplicate the lookup,
|
||||
// but the session ID is derived deterministically from channel+thread
|
||||
// so they converge on the same value and Save is an idempotent upsert.
|
||||
sessions, err := t.sessions.GetByChannel(ctx, platform, channelID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve session: %w", err)
|
||||
@@ -43,14 +54,13 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
||||
// Find the most recent session matching this thread and bot.
|
||||
for _, s := range sessions {
|
||||
if s.Source.BotUser == botUser && s.Source.ThreadID == threadID {
|
||||
t.active[key] = s.SessionID
|
||||
return s.SessionID, nil
|
||||
return t.cacheActive(key, s.SessionID), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new session with a deterministic key so it matches the
|
||||
// legacy sessionKey format used by the LLM responder.
|
||||
id := sessionKeyFromFields(channelID, threadID)
|
||||
id = sessionKeyFromFields(channelID, threadID)
|
||||
now := time.Now()
|
||||
sc := SessionContext{
|
||||
SessionID: id,
|
||||
@@ -66,8 +76,20 @@ func (t *sessionTracker) resolve(ctx context.Context, platform, botUser, channel
|
||||
if err := t.sessions.Save(ctx, sc); err != nil {
|
||||
return "", fmt.Errorf("create session: %w", err)
|
||||
}
|
||||
t.active[key] = id
|
||||
return id, nil
|
||||
return t.cacheActive(key, id), nil
|
||||
}
|
||||
|
||||
// cacheActive records the resolved session for a channel+thread. A racing
|
||||
// caller may have cached one first; its value wins so every caller returns
|
||||
// the same session ID.
|
||||
func (t *sessionTracker) cacheActive(key, sessionID string) string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if existing, ok := t.active[key]; ok {
|
||||
return existing
|
||||
}
|
||||
t.active[key] = sessionID
|
||||
return sessionID
|
||||
}
|
||||
|
||||
func sessionKeyFromFields(channelID, threadID string) string {
|
||||
@@ -77,13 +99,18 @@ func sessionKeyFromFields(channelID, threadID string) string {
|
||||
return channelID + ":" + threadID
|
||||
}
|
||||
|
||||
// setActive sets the current session for a channel+thread.
|
||||
// setActive sets the current session for a channel+thread, overriding any
|
||||
// previously resolved session (used by /new, /branch and /topic).
|
||||
func (t *sessionTracker) setActive(channelID, threadID, sessionID string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.active[sessionTrackerKey(channelID, threadID)] = sessionID
|
||||
}
|
||||
|
||||
// getActive returns the current session ID for a channel+thread.
|
||||
func (t *sessionTracker) getActive(channelID, threadID string) string {
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return t.active[sessionTrackerKey(channelID, threadID)]
|
||||
}
|
||||
|
||||
@@ -265,10 +292,16 @@ func (r *Router) handleRetry(ctx context.Context, msg Message) (string, error) {
|
||||
return "Nothing to retry — no messages remain.", nil
|
||||
}
|
||||
|
||||
lastUserMsg := msgs[0]
|
||||
if r.LLM == nil {
|
||||
return "LLM is not configured. The last response has been removed; send another message to continue.", nil
|
||||
}
|
||||
// Replay the stored text, but route it with the live message's
|
||||
// addressing. Stored history carries no channel or thread -- a session
|
||||
// already is one -- so replaying the stored value verbatim would
|
||||
// resolve to an empty session key and strand the reply.
|
||||
lastUserMsg := msgs[0]
|
||||
lastUserMsg.ChannelID = msg.ChannelID
|
||||
lastUserMsg.ThreadID = msg.ThreadID
|
||||
return r.LLM(ctx, lastUserMsg)
|
||||
}
|
||||
|
||||
@@ -476,6 +509,12 @@ func (r *Router) compressPreview(ctx context.Context, sessionID string) (string,
|
||||
), nil
|
||||
}
|
||||
|
||||
// compressedUserFrom is the sender recorded for user turns that survive
|
||||
// compression. The original username is not recoverable from a
|
||||
// CompressedMessage, and consumers only test From against the bot identity,
|
||||
// so any non-bot value reconstructs the user role correctly.
|
||||
const compressedUserFrom = "user"
|
||||
|
||||
func (r *Router) messagesToCompressed(msgs []Message) []CompressedMessage {
|
||||
compressed := make([]CompressedMessage, 0, len(msgs))
|
||||
for _, h := range msgs {
|
||||
@@ -492,7 +531,14 @@ func (r *Router) messagesToCompressed(msgs []Message) []CompressedMessage {
|
||||
}
|
||||
|
||||
func (r *Router) applyCompress(ctx context.Context, sessionID string, cfg CompressionConfig) (string, error) {
|
||||
msgs, err := r.sessionTracker.sessions.RecentMessages(ctx, sessionID, 200)
|
||||
// Read the whole history: the replacement below becomes the session's
|
||||
// entire history, so summarising only a recent window would silently
|
||||
// discard everything older than it.
|
||||
count, err := r.sessionTracker.sessions.MessageCount(ctx, sessionID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("count messages: %w", err)
|
||||
}
|
||||
msgs, err := r.sessionTracker.sessions.RecentMessages(ctx, sessionID, count)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("load messages: %w", err)
|
||||
}
|
||||
@@ -503,20 +549,28 @@ func (r *Router) applyCompress(ctx context.Context, sessionID string, cfg Compre
|
||||
return fmt.Sprintf("Compression not needed (%d messages, ~%d tokens).", len(msgs), view.TokensBefore), nil
|
||||
}
|
||||
|
||||
// Replace messages with the compressed view. Delete all, then
|
||||
// re-save the compressed messages.
|
||||
allCount, _ := r.sessionTracker.sessions.MessageCount(ctx, sessionID)
|
||||
_, _ = r.sessionTracker.sessions.DeleteRecentMessages(ctx, sessionID, allCount)
|
||||
|
||||
// Replace the history with the compressed view in one store-level
|
||||
// operation. ReplaceMessages writes the replacement before removing
|
||||
// the originals, so a failure leaves recoverable duplicates instead of
|
||||
// wiping the session -- the previous delete-then-save order lost the
|
||||
// entire history if anything went wrong in between.
|
||||
// Write back sender identities, not role labels. Consumers reconstruct
|
||||
// the role by comparing From against the bot's identity, so persisting
|
||||
// the literal string "assistant" would make every retained bot turn
|
||||
// replay to the model as a user turn.
|
||||
compressedMsgs := make([]Message, 0, len(view.Messages))
|
||||
for _, cm := range view.Messages {
|
||||
from := compressedUserFrom
|
||||
if cm.Role == "assistant" {
|
||||
from = r.Identity
|
||||
}
|
||||
compressedMsgs = append(compressedMsgs, Message{
|
||||
From: cm.Role,
|
||||
From: from,
|
||||
Text: cm.Content,
|
||||
})
|
||||
}
|
||||
if err := r.sessionTracker.sessions.SaveMessages(ctx, sessionID, compressedMsgs); err != nil {
|
||||
return "", fmt.Errorf("save compressed messages: %w", err)
|
||||
if err := r.sessionTracker.sessions.ReplaceMessages(ctx, sessionID, compressedMsgs); err != nil {
|
||||
return "", fmt.Errorf("replace with compressed messages: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
|
||||
@@ -3,6 +3,7 @@ package gateway
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -134,6 +135,13 @@ func (f *fakeSessionStore) SaveMessages(_ context.Context, sessionID string, msg
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeSessionStore) ReplaceMessages(_ context.Context, sessionID string, msgs []Message) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.messages[sessionID] = slices.Clone(msgs)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeSessionStore) SearchMessages(_ context.Context, sessionID string, q MessageQuery) (MessagePage, error) {
|
||||
return MessagePage{}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestSessionTrackerConcurrentResolve pins that the active-session map is
|
||||
// safe under the concurrent dispatch the Telegram gateway actually uses:
|
||||
// go-telegram/bot runs each update in its own goroutine. Run with -race.
|
||||
func TestSessionTrackerConcurrentResolve(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
channels int
|
||||
workers int
|
||||
}{
|
||||
{"single channel contended", 1, 32},
|
||||
{"many channels", 16, 32},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
tracker := newSessionTracker(store)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range tc.workers {
|
||||
wg.Go(func() {
|
||||
channel := fmt.Sprintf("chan-%d", i%tc.channels)
|
||||
if _, err := tracker.resolve(ctx, "telegram", "bot", channel, ""); err != nil {
|
||||
t.Errorf("resolve: %v", err)
|
||||
}
|
||||
tracker.setActive(channel, "", "override-"+channel)
|
||||
_ = tracker.getActive(channel, "")
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionTrackerResolveIsStable pins that resolve returns the same
|
||||
// session for the same channel+thread across concurrent callers, so
|
||||
// interleaved messages cannot land in different sessions.
|
||||
func TestSessionTrackerResolveIsStable(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
tracker := newSessionTracker(store)
|
||||
|
||||
const workers = 32
|
||||
ids := make([]string, workers)
|
||||
var wg sync.WaitGroup
|
||||
for i := range workers {
|
||||
wg.Go(func() {
|
||||
id, err := tracker.resolve(ctx, "telegram", "bot", "chan", "thread")
|
||||
if err != nil {
|
||||
t.Errorf("resolve: %v", err)
|
||||
return
|
||||
}
|
||||
ids[i] = id
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
for i, id := range ids {
|
||||
if id != ids[0] {
|
||||
t.Fatalf("worker %d resolved %q, worker 0 resolved %q", i, id, ids[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCanonicalMessageIDIsSessionScoped pins that the same upstream message
|
||||
// copied into another session gets its own identity, so branch lineage does
|
||||
// not alias across conversations.
|
||||
func TestCanonicalMessageIDIsSessionScoped(t *testing.T) {
|
||||
a := newMessageID("session-a", "tg-1")
|
||||
b := newMessageID("session-b", "tg-1")
|
||||
if a == b {
|
||||
t.Errorf("sessions a and b both derived %q for the same upstream ID", a)
|
||||
}
|
||||
if again := newMessageID("session-a", "tg-1"); again != a {
|
||||
t.Errorf("derivation is not deterministic: %q then %q", a, again)
|
||||
}
|
||||
if first, second := newMessageID("session-a", ""), newMessageID("session-a", ""); first == second {
|
||||
t.Error("messages with no upstream ID must get distinct identities")
|
||||
}
|
||||
}
|
||||
@@ -7,179 +7,18 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// base is a fixed instant so message timestamps are deterministic; tests
|
||||
// derive every other instant from it rather than calling time.Now.
|
||||
var base = time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func at(offset time.Duration) time.Time { return base.Add(offset) }
|
||||
|
||||
func msg(text string, offset time.Duration) Message {
|
||||
return Message{From: "user", Text: text, At: at(offset)}
|
||||
}
|
||||
|
||||
func texts(msgs []Message) []string {
|
||||
out := make([]string, 0, len(msgs))
|
||||
for _, m := range msgs {
|
||||
out = append(out, m.Text)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func equal(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// TestSaveMessagesAppends pins the data-loss defect: a bulk save into a
|
||||
// session that already holds messages must append, never overwrite.
|
||||
func TestSaveMessagesAppends(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
existing []string
|
||||
bulk []string
|
||||
want []string
|
||||
}{
|
||||
{"bulk into empty", nil, []string{"a", "b"}, []string{"a", "b"}},
|
||||
{"bulk into non-empty", []string{"m0", "m1"}, []string{"a", "b"}, []string{"m0", "m1", "a", "b"}},
|
||||
{"single bulk into non-empty", []string{"m0"}, []string{"a"}, []string{"m0", "a"}},
|
||||
{"empty bulk is a no-op", []string{"m0", "m1"}, nil, []string{"m0", "m1"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
var offset time.Duration
|
||||
for _, text := range tc.existing {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(text, offset)); err != nil {
|
||||
t.Fatalf("SaveMessage(%q): %v", text, err)
|
||||
}
|
||||
offset += time.Second
|
||||
}
|
||||
|
||||
bulk := make([]Message, 0, len(tc.bulk))
|
||||
for _, text := range tc.bulk {
|
||||
bulk = append(bulk, msg(text, offset))
|
||||
offset += time.Second
|
||||
}
|
||||
if err := s.SaveMessages(ctx, "sess", bulk); err != nil {
|
||||
t.Fatalf("SaveMessages: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.RecentMessages(ctx, "sess", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages: %v", err)
|
||||
}
|
||||
if !equal(texts(got), tc.want) {
|
||||
t.Errorf("got %v, want %v", texts(got), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteThenSaveDoesNotReuseKeys pins the seq-reuse defect: deleting
|
||||
// the newest messages then saving must not collide with a live key.
|
||||
func TestDeleteThenSaveDoesNotReuseKeys(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
initial int
|
||||
deleteCount int
|
||||
thenSave []string
|
||||
want []string
|
||||
}{
|
||||
{"delete one, save one", 3, 1, []string{"new"}, []string{"m0", "m1", "new"}},
|
||||
{"delete two, save two", 4, 2, []string{"x", "y"}, []string{"m0", "m1", "x", "y"}},
|
||||
{"delete all, save one", 2, 2, []string{"only"}, []string{"only"}},
|
||||
{"delete more than exist", 2, 5, []string{"only"}, []string{"only"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
var offset time.Duration
|
||||
for i := range tc.initial {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(fmt.Sprintf("m%d", i), offset)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
offset += time.Second
|
||||
}
|
||||
if _, err := s.DeleteRecentMessages(ctx, "sess", tc.deleteCount); err != nil {
|
||||
t.Fatalf("DeleteRecentMessages: %v", err)
|
||||
}
|
||||
for _, text := range tc.thenSave {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(text, offset)); err != nil {
|
||||
t.Fatalf("SaveMessage(%q): %v", text, err)
|
||||
}
|
||||
offset += time.Second
|
||||
}
|
||||
|
||||
got, err := s.RecentMessages(ctx, "sess", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages: %v", err)
|
||||
}
|
||||
if !equal(texts(got), tc.want) {
|
||||
t.Errorf("got %v, want %v", texts(got), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessagesOrderByTimestamp pins chronology: ordering follows Message.At,
|
||||
// not the order rows happened to be written.
|
||||
func TestMessagesOrderByTimestamp(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
writeOrder []time.Duration
|
||||
want []string
|
||||
}{
|
||||
{"already chronological", []time.Duration{0, time.Second, 2 * time.Second}, []string{"t0", "t1", "t2"}},
|
||||
{"written out of order", []time.Duration{2 * time.Second, 0, time.Second}, []string{"t1", "t2", "t0"}},
|
||||
{"reverse written", []time.Duration{2 * time.Second, time.Second, 0}, []string{"t2", "t1", "t0"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
for i, offset := range tc.writeOrder {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(fmt.Sprintf("t%d", i), offset)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
got, err := s.RecentMessages(ctx, "sess", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages: %v", err)
|
||||
}
|
||||
// want lists texts in the order their timestamps dictate.
|
||||
if !equal(texts(got), tc.want) {
|
||||
t.Errorf("got %v, want %v", texts(got), tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageTimestampRoundTrips pins that At survives persistence, since
|
||||
// the previous implementation wrote a timestamp and never read it back.
|
||||
// Resolution is milliseconds: _ts is stored as Unix milliseconds so the
|
||||
// text index does not tokenize it. This is a NellDB-specific persistence
|
||||
// detail rather than a cross-implementation contract, so it stays here
|
||||
// rather than in the shared SessionStore suite.
|
||||
func TestMessageTimestampRoundTrips(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
want := at(90 * time.Minute)
|
||||
want := at(90*time.Minute + 123*time.Millisecond)
|
||||
if err := s.SaveMessage(ctx, "sess", Message{From: "u", Text: "hello", At: want}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
@@ -195,231 +34,12 @@ func TestMessageTimestampRoundTrips(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSaveMessageStampsMissingTimestamp pins that a caller omitting At still
|
||||
// gets a usable ordering key rather than the zero time.
|
||||
func TestSaveMessageStampsMissingTimestamp(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
before := time.Now().UTC()
|
||||
if err := s.SaveMessage(ctx, "sess", Message{From: "u", Text: "no timestamp"}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
after := time.Now().UTC()
|
||||
|
||||
got, err := s.RecentMessages(ctx, "sess", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d messages, want 1", len(got))
|
||||
}
|
||||
if got[0].At.Before(before) || got[0].At.After(after) {
|
||||
t.Errorf("At = %s, want within [%s, %s]", got[0].At, before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionsAreIsolated pins per-session collection scoping: one session's
|
||||
// history and search results must never include another's.
|
||||
func TestSessionsAreIsolated(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
if err := s.SaveMessage(ctx, "alpha", msg("alpha secret", 0)); err != nil {
|
||||
t.Fatalf("SaveMessage alpha: %v", err)
|
||||
}
|
||||
if err := s.SaveMessage(ctx, "beta", msg("beta secret", time.Second)); err != nil {
|
||||
t.Fatalf("SaveMessage beta: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct{ session, want string }{
|
||||
{"alpha", "alpha secret"},
|
||||
{"beta", "beta secret"},
|
||||
} {
|
||||
got, err := s.RecentMessages(ctx, tc.session, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages(%s): %v", tc.session, err)
|
||||
}
|
||||
if !equal(texts(got), []string{tc.want}) {
|
||||
t.Errorf("session %s history = %v, want [%s]", tc.session, texts(got), tc.want)
|
||||
}
|
||||
|
||||
page, err := s.SearchMessages(ctx, tc.session, MessageQuery{Query: "secret", Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages(%s): %v", tc.session, err)
|
||||
}
|
||||
if !equal(texts(page.Messages), []string{tc.want}) {
|
||||
t.Errorf("session %s search = %v, want [%s]", tc.session, texts(page.Messages), tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
count, err := s.MessageCount(ctx, "alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Errorf("MessageCount(alpha) = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchMessagesPaging pins that Limit is a page size, not a total cap,
|
||||
// and that paging walks the whole result set without gaps or repeats.
|
||||
func TestSearchMessagesPaging(t *testing.T) {
|
||||
const total = 250 // deliberately above the old hard-coded 200 ceiling
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
pageSize int
|
||||
wantPages int
|
||||
}{
|
||||
{"page size 10", 10, 25},
|
||||
{"page size 50", 50, 5},
|
||||
{"page size larger than result set", 500, 1},
|
||||
{"page size 1", 1, total},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
for i := range total {
|
||||
m := msg(fmt.Sprintf("needle %d", i), time.Duration(i)*time.Second)
|
||||
if err := s.SaveMessage(ctx, "sess", m); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
pages, offset := 0, 0
|
||||
for {
|
||||
page, err := s.SearchMessages(ctx, "sess", MessageQuery{
|
||||
Query: "needle", Limit: tc.pageSize, Offset: offset,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages: %v", err)
|
||||
}
|
||||
pages++
|
||||
if len(page.Messages) > tc.pageSize {
|
||||
t.Fatalf("page returned %d messages, exceeds page size %d", len(page.Messages), tc.pageSize)
|
||||
}
|
||||
for _, m := range page.Messages {
|
||||
if seen[m.Text] {
|
||||
t.Errorf("duplicate result across pages: %q", m.Text)
|
||||
}
|
||||
seen[m.Text] = true
|
||||
}
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
if page.NextOffset <= offset {
|
||||
t.Fatalf("NextOffset %d did not advance past %d", page.NextOffset, offset)
|
||||
}
|
||||
offset = page.NextOffset
|
||||
if pages > total+1 {
|
||||
t.Fatal("paging did not terminate")
|
||||
}
|
||||
}
|
||||
|
||||
if pages != tc.wantPages {
|
||||
t.Errorf("walked %d pages, want %d", pages, tc.wantPages)
|
||||
}
|
||||
if len(seen) != total {
|
||||
t.Errorf("saw %d unique messages across all pages, want %d", len(seen), total)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchMessagesBeyondOldCeiling pins the specific regression: a match
|
||||
// older than the most recent 200 messages must still be findable.
|
||||
func TestSearchMessagesBeyondOldCeiling(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
// The needle is the oldest message; 300 newer messages bury it well
|
||||
// past the old 200-message window.
|
||||
if err := s.SaveMessage(ctx, "sess", msg("distinctive needle", 0)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
for i := range 300 {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(fmt.Sprintf("filler %d", i), time.Duration(i+1)*time.Second)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
page, err := s.SearchMessages(ctx, "sess", MessageQuery{Query: "distinctive", Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages: %v", err)
|
||||
}
|
||||
if !equal(texts(page.Messages), []string{"distinctive needle"}) {
|
||||
t.Errorf("got %v, want [distinctive needle]", texts(page.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
// TestHistoryBeyondScanPageLimit pins that history operations page past
|
||||
// NellDB's internal 1000-row ScanSince clamp instead of silently truncating.
|
||||
// /compress calls DeleteRecentMessages with the full message count, so a
|
||||
// silent cap there strands messages while reporting success.
|
||||
func TestHistoryBeyondScanPageLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
total int
|
||||
}{
|
||||
{"just under the clamp", 999},
|
||||
{"exactly the clamp", 1000},
|
||||
{"over the clamp", 1200},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
for i := range tc.total {
|
||||
m := msg(fmt.Sprintf("m%d", i), time.Duration(i)*time.Second)
|
||||
if err := s.SaveMessage(ctx, "sess", m); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := s.RecentMessages(ctx, "sess", tc.total)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages: %v", err)
|
||||
}
|
||||
if len(got) != tc.total {
|
||||
t.Errorf("RecentMessages returned %d, want %d", len(got), tc.total)
|
||||
}
|
||||
if len(got) > 0 && got[0].Text != "m0" {
|
||||
t.Errorf("oldest message = %q, want m0", got[0].Text)
|
||||
}
|
||||
|
||||
deleted, err := s.DeleteRecentMessages(ctx, "sess", tc.total)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteRecentMessages: %v", err)
|
||||
}
|
||||
if deleted != tc.total {
|
||||
t.Errorf("deleted %d, want %d", deleted, tc.total)
|
||||
}
|
||||
remaining, err := s.MessageCount(ctx, "sess")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount: %v", err)
|
||||
}
|
||||
if remaining != 0 {
|
||||
t.Errorf("%d messages remain after deleting all", remaining)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchTruncationIsReported pins that hitting the engine's ranked-result
|
||||
// ceiling is surfaced as Truncated rather than passed off as "no more
|
||||
// results", so a caller can tell an exhausted result set from a cut-off one.
|
||||
// MaxSearchResults is a NellDB engine ceiling (nl.MaxTextSearchLimit), so
|
||||
// this test is pinned against the NellDB implementation rather than the
|
||||
// shared cross-implementation suite.
|
||||
func TestSearchTruncationIsReported(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
@@ -488,25 +108,101 @@ func TestSearchExhaustedIsNotTruncated(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteSessionDropsMessages pins that removing a session also removes
|
||||
// its message collection rather than orphaning it.
|
||||
func TestDeleteSessionDropsMessages(t *testing.T) {
|
||||
// TestTruncatedReflectsResultSetNotRequest pins that a deep offset over a
|
||||
// small result set is reported as exhausted, not truncated. Anchored on
|
||||
// MaxSearchResults, the NellDB engine ceiling.
|
||||
func TestTruncatedReflectsResultSetNotRequest(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
matches int
|
||||
offset int
|
||||
limit int
|
||||
wantTruncated bool
|
||||
}{
|
||||
{"deep offset over tiny result set", 3, MaxSearchResults, 50, false},
|
||||
{"deep offset over empty result set", 0, MaxSearchResults, 50, false},
|
||||
{"shallow offset over small result set", 3, 0, 50, false},
|
||||
{"offset just past a small result set", 3, 10, 50, false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
for i := range tc.matches {
|
||||
if err := s.SaveMessage(ctx, "sess", msg(fmt.Sprintf("needle %d", i), time.Duration(i)*time.Second)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
page, err := s.SearchMessages(ctx, "sess", MessageQuery{
|
||||
Query: "needle", Limit: tc.limit, Offset: tc.offset,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages: %v", err)
|
||||
}
|
||||
if page.Truncated != tc.wantTruncated {
|
||||
t.Errorf("Truncated = %v, want %v", page.Truncated, tc.wantTruncated)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageDBCacheIsBounded pins that touching many sessions does not grow
|
||||
// the DocDB cache without limit, and that eviction costs nothing but time.
|
||||
// This exercises nellSessionStore internals directly, so it is pinned
|
||||
// against the NellDB implementation rather than the shared suite.
|
||||
func TestMessageDBCacheIsBounded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := NewSessionStoreMemory("test-node")
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
if err := s.SaveMessage(ctx, "sess", msg("hello", 0)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
if err := s.Delete(ctx, "sess"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
const sessions = maxCachedMessageDBs * 3
|
||||
for i := range sessions {
|
||||
id := fmt.Sprintf("sess-%d", i)
|
||||
if err := s.SaveMessage(ctx, id, msg(fmt.Sprintf("in %s", id), time.Duration(i)*time.Second)); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
count, err := s.MessageCount(ctx, "sess")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount: %v", err)
|
||||
ns, ok := s.(*nellSessionStore)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want *nellSessionStore", s)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("MessageCount after Delete = %d, want 0", count)
|
||||
ns.mu.Lock()
|
||||
cached, ordered := len(ns.msgDBs), len(ns.msgOrder)
|
||||
ns.mu.Unlock()
|
||||
|
||||
if cached > maxCachedMessageDBs {
|
||||
t.Errorf("cache holds %d entries, want at most %d", cached, maxCachedMessageDBs)
|
||||
}
|
||||
if cached != ordered {
|
||||
t.Errorf("cache map has %d entries but LRU order has %d", cached, ordered)
|
||||
}
|
||||
|
||||
// An evicted session must still read back correctly.
|
||||
got, err := s.RecentMessages(ctx, "sess-0", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages after eviction: %v", err)
|
||||
}
|
||||
if !equal(texts(got), []string{"in sess-0"}) {
|
||||
t.Errorf("evicted session read back as %v, want [in sess-0]", texts(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestMessageIDsAreUnique pins that generated identities do not collide.
|
||||
// Ordering no longer rests on ID shape -- timestamps are strictly
|
||||
// increasing within a session -- but identity must still be unique, and the
|
||||
// previous counter-based scheme could wrap and reuse a key.
|
||||
func TestMessageIDsAreUnique(t *testing.T) {
|
||||
const count = 10_000
|
||||
seen := make(map[string]struct{}, count)
|
||||
for range count {
|
||||
id := newMessageID("sess", "")
|
||||
if _, dup := seen[id]; dup {
|
||||
t.Fatalf("duplicate message ID %q", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
nl "github.com/samcharles93/NellDB"
|
||||
"github.com/samcharles93/NellDB/sdk"
|
||||
)
|
||||
@@ -68,6 +68,12 @@ type MessageHistory interface {
|
||||
// SaveMessages bulk-saves messages for branch inheritance.
|
||||
SaveMessages(ctx context.Context, sessionID string, msgs []Message) error
|
||||
|
||||
// ReplaceMessages makes msgs the session's entire history. It is the
|
||||
// /compress primitive. The replacement is written before the previous
|
||||
// history is removed, so an interrupted call leaves both sets
|
||||
// present -- recoverable duplicates -- rather than an empty session.
|
||||
ReplaceMessages(ctx context.Context, sessionID string, msgs []Message) error
|
||||
|
||||
// SearchMessages returns one page of messages matching the query,
|
||||
// searching the session's entire history rather than a recent window.
|
||||
SearchMessages(ctx context.Context, sessionID string, q MessageQuery) (MessagePage, error)
|
||||
@@ -80,8 +86,9 @@ type MessageHistory interface {
|
||||
// Limit is a page size, not a cap on how much history is considered: the
|
||||
// whole session is searched and the matches are returned one page at a time.
|
||||
type MessageQuery struct {
|
||||
// Query is the free-text search. Terms are matched against message
|
||||
// text; every term must be present.
|
||||
// Query is the free-text search. Every term must be present. Terms
|
||||
// match a message's text and its sender, so searching a username
|
||||
// returns that user's messages.
|
||||
Query string
|
||||
// Limit is the maximum number of messages in one page. Zero or
|
||||
// negative selects DefaultMessagePageSize; values above
|
||||
@@ -98,14 +105,24 @@ type MessagePage struct {
|
||||
Messages []Message
|
||||
// NextOffset is the Offset that returns the following page. It is
|
||||
// only meaningful when HasMore is true.
|
||||
//
|
||||
// Paging is by offset over a relevance-ranked list, and the ranking
|
||||
// uses collection-wide term statistics, so a message saved between
|
||||
// two page fetches re-scores the whole list. A walk across an active
|
||||
// session can therefore skip or repeat a result; treat the pages as a
|
||||
// snapshot-in-motion, not a transactional cursor.
|
||||
NextOffset int
|
||||
// HasMore reports whether further pages exist.
|
||||
HasMore bool
|
||||
// Truncated reports that paging reached MaxSearchResults. Further
|
||||
// matches exist in the session but the engine's ranked result set
|
||||
// cannot be paged beyond that ceiling, so HasMore is false even
|
||||
// though the result set is incomplete. Callers should narrow the
|
||||
// Truncated reports that paging reached MaxSearchResults, so HasMore
|
||||
// is false even though matches may remain. Callers should narrow the
|
||||
// query rather than page further.
|
||||
//
|
||||
// It is conservative at the boundary: a query matching exactly
|
||||
// MaxSearchResults documents is indistinguishable from one matching
|
||||
// more, because the engine returns a ranked slice with no total and no
|
||||
// cursor. Such a query reports Truncated even though every match was
|
||||
// in fact returned.
|
||||
Truncated bool
|
||||
}
|
||||
|
||||
@@ -130,10 +147,14 @@ type nellSessionStore struct {
|
||||
nodeID string
|
||||
mu sync.Mutex
|
||||
// msgDBs caches one DocDB per session message collection, keyed by
|
||||
// session ID. Guarded by mu.
|
||||
// session ID. Guarded by mu, bounded by maxCachedMessageDBs.
|
||||
msgDBs map[string]*sdk.DocDB
|
||||
// msgSeq disambiguates message IDs written within the same nanosecond.
|
||||
msgSeq atomic.Uint64
|
||||
// msgOrder is msgDBs' LRU order, least-recently-used first.
|
||||
msgOrder []string
|
||||
// lastTS caches the newest persisted timestamp per session, in Unix
|
||||
// milliseconds, so the monotonic clamp costs one store read per
|
||||
// session rather than one per write. Guarded by mu.
|
||||
lastTS map[string]int64
|
||||
}
|
||||
|
||||
// NewSessionStore creates a SessionStore backed by an existing NellDB
|
||||
@@ -144,6 +165,7 @@ func NewSessionStore(st nl.Store, nodeID string) SessionStore {
|
||||
store: st,
|
||||
nodeID: nodeID,
|
||||
msgDBs: make(map[string]*sdk.DocDB),
|
||||
lastTS: make(map[string]int64),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,26 +220,24 @@ func (s *nellSessionStore) Delete(ctx context.Context, sessionID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
_, err := s.db.Remove(ctx, sessionID)
|
||||
if err != nil && !errors.Is(err, sdk.ErrNotFound) {
|
||||
return fmt.Errorf("sessionstore: delete: %w", err)
|
||||
}
|
||||
// Drop the session's message collection too, so deleting a session does
|
||||
// not orphan its history.
|
||||
// Messages go first. If this fails partway the session document still
|
||||
// points at the surviving history, so a retry can finish the job;
|
||||
// removing the session first would strand those messages unreachable.
|
||||
db := s.messagesFor(sessionID)
|
||||
result, err := db.AllDocs(ctx, sdk.DocRange{IncludeDocs: false})
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: delete messages: %w", err)
|
||||
}
|
||||
for _, row := range result.Rows {
|
||||
if isMetaKey(row.ID) {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Remove(ctx, row.ID); err != nil && !errors.Is(err, sdk.ErrNotFound) {
|
||||
return fmt.Errorf("sessionstore: delete message %s: %w", row.ID, err)
|
||||
}
|
||||
}
|
||||
delete(s.msgDBs, sessionID)
|
||||
s.dropMsgDB(sessionID)
|
||||
|
||||
if _, err := s.db.Remove(ctx, sessionID); err != nil && !errors.Is(err, sdk.ErrNotFound) {
|
||||
return fmt.Errorf("sessionstore: delete: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -338,53 +358,210 @@ func msgCollection(sessionID string) string {
|
||||
return "messages:" + sessionID
|
||||
}
|
||||
|
||||
// maxCachedMessageDBs bounds the per-session DocDB cache. Each entry holds
|
||||
// a rev map with one entry per live message in that collection, so an
|
||||
// unbounded cache would grow with every session the daemon ever touches.
|
||||
const maxCachedMessageDBs = 64
|
||||
|
||||
// messagesFor returns the cached DocDB for a session's message collection,
|
||||
// creating it on first use. Callers must hold s.mu: sdk.New reindexes the
|
||||
// collection from the log store, so the cache keeps repeat access cheap.
|
||||
// The cache is LRU-bounded; eviction only drops a rev cache that the next
|
||||
// sdk.New rebuilds, so it costs time rather than correctness.
|
||||
func (s *nellSessionStore) messagesFor(sessionID string) *sdk.DocDB {
|
||||
if db, ok := s.msgDBs[sessionID]; ok {
|
||||
s.touchMsgDB(sessionID)
|
||||
return db
|
||||
}
|
||||
db := sdk.New(s.store, s.nodeID, msgCollection(sessionID))
|
||||
s.msgDBs[sessionID] = db
|
||||
s.msgOrder = append(s.msgOrder, sessionID)
|
||||
for len(s.msgOrder) > maxCachedMessageDBs {
|
||||
evict := s.msgOrder[0]
|
||||
s.msgOrder = s.msgOrder[1:]
|
||||
delete(s.msgDBs, evict)
|
||||
// lastTS is re-read from the store on next use.
|
||||
delete(s.lastTS, evict)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// touchMsgDB moves a session to the most-recently-used end of the cache.
|
||||
func (s *nellSessionStore) touchMsgDB(sessionID string) {
|
||||
if i := slices.Index(s.msgOrder, sessionID); i >= 0 {
|
||||
s.msgOrder = append(slices.Delete(s.msgOrder, i, i+1), sessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// dropMsgDB removes a session from the cache entirely.
|
||||
func (s *nellSessionStore) dropMsgDB(sessionID string) {
|
||||
delete(s.msgDBs, sessionID)
|
||||
delete(s.lastTS, sessionID)
|
||||
if i := slices.Index(s.msgOrder, sessionID); i >= 0 {
|
||||
s.msgOrder = slices.Delete(s.msgOrder, i, i+1)
|
||||
}
|
||||
}
|
||||
|
||||
// msgID builds a time-ordered, collision-free document ID. Ordering is
|
||||
// driven by the _ts field, but a matching ID order keeps equal-timestamp
|
||||
// ties stable across scans and cursors. The counter disambiguates messages
|
||||
// that land within the same nanosecond.
|
||||
func (s *nellSessionStore) msgID(at time.Time) string {
|
||||
return fmt.Sprintf("%020d-%08d", at.UnixNano(), s.msgSeq.Add(1))
|
||||
// that land within the same nanosecond; it resets on restart, so uniqueness
|
||||
// across restarts rests on the nanosecond timestamp.
|
||||
// messageIDNamespace scopes the deterministic derivation below. It is a
|
||||
// fixed, arbitrary UUID: only its stability matters.
|
||||
var messageIDNamespace = uuid.MustParse("6f8d2b1e-9c4a-4f37-8a56-1d0e7b3c9a42")
|
||||
|
||||
// newMessageID returns the canonical, application-generated identifier a
|
||||
// message is stored under.
|
||||
//
|
||||
// Platform identifiers are correlation metadata, never the canonical ID, so
|
||||
// a channel-native ID is not used as the key. It is instead an input to a
|
||||
// deterministic derivation scoped to the session, which keeps the canonical
|
||||
// ID archie's own while still making redelivery of the same upstream
|
||||
// message resolve to the same record. Scoping by session means the same
|
||||
// upstream message copied into a branch gets its own identity.
|
||||
//
|
||||
// Messages with no upstream identity get a random ID and always append.
|
||||
func newMessageID(sessionID, sourceID string) string {
|
||||
if sourceID == "" {
|
||||
return uuid.NewString()
|
||||
}
|
||||
return uuid.NewSHA1(messageIDNamespace, []byte(sessionID+"\x00"+sourceID)).String()
|
||||
}
|
||||
|
||||
// messageToDoc renders a message for persistence. at must already be
|
||||
// resolved to a concrete instant.
|
||||
func (s *nellSessionStore) messageToDoc(msg Message, at time.Time) sdk.Doc {
|
||||
return sdk.Doc{
|
||||
sdk.FieldID: s.msgID(at),
|
||||
//
|
||||
// Only fields that should be findable by SearchMessages belong here as
|
||||
// strings: NellDB's text index tokenizes every string value in the payload
|
||||
// except a small reserved set, so an RFC3339 _ts would make "2026" match
|
||||
// every message in the session. _ts is therefore written as Unix
|
||||
// milliseconds, which Record.Timestamp accepts and the indexer ignores
|
||||
// because it is a number. Channel and thread are session-level constants
|
||||
// (a session is a platform/bot/channel/thread tuple) and are deliberately
|
||||
// not persisted per message. source_id is persisted per message because it
|
||||
// is the external correlation metadata an export must carry, and it is only
|
||||
// written when present: a message with no upstream identity produces the
|
||||
// same document it always did. It is stored as the integer values of its
|
||||
// UTF-8 bytes rather than as a string, because the indexer ignores numbers
|
||||
// -- a plain string would make external correlation IDs match
|
||||
// SearchMessages. The byte array survives the log store's JSON round-trip.
|
||||
// messageID must already be the canonical ID the message is stored under.
|
||||
func messageToDoc(messageID string, msg Message, at time.Time) sdk.Doc {
|
||||
doc := sdk.Doc{
|
||||
sdk.FieldID: messageID,
|
||||
// NellDB orders scans on _ts, preferring it over the record HLC.
|
||||
nl.TimestampField: at.Format(time.RFC3339Nano),
|
||||
nl.TimestampField: at.UnixMilli(),
|
||||
"from": msg.From,
|
||||
"text": msg.Text,
|
||||
"channel_id": msg.ChannelID,
|
||||
"thread_id": msg.ThreadID,
|
||||
}
|
||||
if msg.SourceID != "" {
|
||||
doc["source_id"] = utf8ByteValues(msg.SourceID)
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func docToMessage(doc sdk.Doc) Message {
|
||||
msg := Message{
|
||||
MessageID: strField(doc, sdk.FieldID),
|
||||
From: strField(doc, "from"),
|
||||
Text: strField(doc, "text"),
|
||||
ChannelID: strField(doc, "channel_id"),
|
||||
ThreadID: strField(doc, "thread_id"),
|
||||
}
|
||||
if v := strField(doc, nl.TimestampField); v != "" {
|
||||
msg.At, _ = time.Parse(time.RFC3339Nano, v)
|
||||
if ms, ok := numField(doc, nl.TimestampField); ok {
|
||||
msg.At = time.UnixMilli(ms).UTC()
|
||||
}
|
||||
// source_id is optional: messages written without one never carry the
|
||||
// field, and documents persisted before it was stored lack it, so a
|
||||
// missing key reads back as "". It is stored as UTF-8 byte values (see
|
||||
// sourceIDFromDoc), never as a string, to keep correlation IDs out of
|
||||
// the text index.
|
||||
msg.SourceID = sourceIDFromDoc(doc)
|
||||
return msg
|
||||
}
|
||||
|
||||
// utf8ByteValues renders s as the integer values of its UTF-8 bytes, the
|
||||
// persisted shape of source_id. Numbers are invisible to NellDB's text
|
||||
// index, so the correlation ID stays out of SearchMessages, and the array
|
||||
// survives the log store's JSON round-trip exactly. []byte is deliberately
|
||||
// not used: JSON would encode it as a base64 string, which is both
|
||||
// searchable and not the original text.
|
||||
func utf8ByteValues(s string) []int {
|
||||
b := []byte(s)
|
||||
out := make([]int, len(b))
|
||||
for i, c := range b {
|
||||
out[i] = int(c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sourceIDFromDoc decodes the persisted source_id representation back into
|
||||
// the original UTF-8 string. It tolerates every shape a read can produce:
|
||||
// []int as written in-process, and []any of float64 after the log store's
|
||||
// JSON round-trip. A missing field -- a document written before source_id
|
||||
// was stored, or a message with no upstream identity -- reads back empty,
|
||||
// as does any unrecognized shape (e.g. a stray string, which would be
|
||||
// searchable and is therefore never echoed).
|
||||
func sourceIDFromDoc(doc sdk.Doc) string {
|
||||
v, ok := doc["source_id"]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
var nums []int64
|
||||
switch t := v.(type) {
|
||||
case []int:
|
||||
nums = make([]int64, len(t))
|
||||
for i, n := range t {
|
||||
nums[i] = int64(n)
|
||||
}
|
||||
case []int64:
|
||||
nums = t
|
||||
case []float64:
|
||||
nums = make([]int64, len(t))
|
||||
for i, n := range t {
|
||||
nums[i] = int64(n)
|
||||
}
|
||||
case []any:
|
||||
nums = make([]int64, len(t))
|
||||
for i, item := range t {
|
||||
var n int64
|
||||
switch x := item.(type) {
|
||||
case float64:
|
||||
n = int64(x)
|
||||
case int64:
|
||||
n = x
|
||||
case int:
|
||||
n = int64(x)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
nums[i] = n
|
||||
}
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
buf := make([]byte, len(nums))
|
||||
for i, n := range nums {
|
||||
buf[i] = byte(n)
|
||||
}
|
||||
return string(buf)
|
||||
}
|
||||
|
||||
// numField reads an integer payload field. JSON round-trips numbers as
|
||||
// float64, but an in-process Doc still holds whatever the writer put there,
|
||||
// so both shapes have to be handled.
|
||||
func numField(doc sdk.Doc, key string) (int64, bool) {
|
||||
switch v := doc[key].(type) {
|
||||
case int64:
|
||||
return v, true
|
||||
case float64:
|
||||
return int64(v), true
|
||||
case int:
|
||||
return int64(v), true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
// stamp resolves a message's application time, defaulting a zero At to now
|
||||
// so every record carries a usable ordering key.
|
||||
func stamp(msg Message) time.Time {
|
||||
@@ -397,15 +574,84 @@ func stamp(msg Message) time.Time {
|
||||
func (s *nellSessionStore) SaveMessage(ctx context.Context, sessionID string, msg Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.saveMessageLocked(ctx, sessionID, msg)
|
||||
_, err := s.saveMessageLocked(ctx, sessionID, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *nellSessionStore) saveMessageLocked(ctx context.Context, sessionID string, msg Message) error {
|
||||
doc := s.messageToDoc(msg, stamp(msg))
|
||||
if _, err := s.messagesFor(sessionID).Put(ctx, doc); err != nil {
|
||||
return fmt.Errorf("sessionstore: save message: %w", err)
|
||||
// saveMessageLocked persists one message and reports the document ID it was
|
||||
// written under. Callers must hold s.mu.
|
||||
func (s *nellSessionStore) saveMessageLocked(ctx context.Context, sessionID string, msg Message) (string, error) {
|
||||
db := s.messagesFor(sessionID)
|
||||
// A message carrying a canonical ID keeps it: re-saving one that was
|
||||
// read back must not mint a new identity, or branch points and related
|
||||
// records would lose their referent.
|
||||
id := msg.MessageID
|
||||
if id == "" {
|
||||
id = newMessageID(sessionID, msg.SourceID)
|
||||
}
|
||||
return nil
|
||||
|
||||
// Stored messages are immutable. An existing record has been seen
|
||||
// before -- a redelivered update, or a re-save of history already
|
||||
// held -- so it stands and the write is a no-op. Representing a later
|
||||
// edit means adding a related record, not rewriting this one.
|
||||
switch _, err := db.Get(ctx, id); {
|
||||
case err == nil:
|
||||
return id, nil
|
||||
case !errors.Is(err, sdk.ErrNotFound):
|
||||
return "", fmt.Errorf("sessionstore: read existing message: %w", err)
|
||||
}
|
||||
|
||||
at, err := s.nextTimestamp(ctx, sessionID, db, stamp(msg))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := db.Put(ctx, messageToDoc(id, msg, at)); err != nil {
|
||||
return "", fmt.Errorf("sessionstore: save message: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// nextTimestamp picks the timestamp a message is persisted under.
|
||||
//
|
||||
// Within a session the value is strictly increasing, which is what makes
|
||||
// history an append-only linear sequence. Channels report time at wildly
|
||||
// different resolutions -- a Telegram message carries whole seconds while a
|
||||
// locally generated reply carries milliseconds -- so honouring the sender's
|
||||
// clock verbatim lets a reply sort ahead of the message that provoked it.
|
||||
// The sender's time is kept whenever it is already ahead, and otherwise
|
||||
// advanced by the smallest representable step.
|
||||
//
|
||||
// Distinct timestamps within a session also mean the document ID never
|
||||
// decides ordering, which is why the canonical ID does not encode time.
|
||||
func (s *nellSessionStore) nextTimestamp(ctx context.Context, sessionID string, db *sdk.DocDB, want time.Time) (time.Time, error) {
|
||||
last, err := s.lastTimestamp(ctx, sessionID, db)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
ms := want.UnixMilli()
|
||||
if ms <= last {
|
||||
ms = last + 1
|
||||
}
|
||||
s.lastTS[sessionID] = ms
|
||||
return time.UnixMilli(ms).UTC(), nil
|
||||
}
|
||||
|
||||
// lastTimestamp returns the newest persisted timestamp in a session, in
|
||||
// Unix milliseconds, reading the store only on the first call per session.
|
||||
func (s *nellSessionStore) lastTimestamp(ctx context.Context, sessionID string, db *sdk.DocDB) (int64, error) {
|
||||
if ms, ok := s.lastTS[sessionID]; ok {
|
||||
return ms, nil
|
||||
}
|
||||
page, err := db.ScanSince(ctx, sdk.MessageRange{Limit: 1, Desc: true})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sessionstore: read newest message: %w", err)
|
||||
}
|
||||
var ms int64
|
||||
if len(page.Rows) > 0 && page.Rows[0].Doc != nil {
|
||||
ms, _ = numField(page.Rows[0].Doc, nl.TimestampField)
|
||||
}
|
||||
s.lastTS[sessionID] = ms
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
func (s *nellSessionStore) SaveMessages(ctx context.Context, sessionID string, msgs []Message) error {
|
||||
@@ -414,13 +660,55 @@ func (s *nellSessionStore) SaveMessages(ctx context.Context, sessionID string, m
|
||||
// Each message gets its own time-ordered ID, so a bulk save appends to
|
||||
// whatever the session already holds instead of overwriting from zero.
|
||||
for _, msg := range msgs {
|
||||
if err := s.saveMessageLocked(ctx, sessionID, msg); err != nil {
|
||||
if _, err := s.saveMessageLocked(ctx, sessionID, msg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *nellSessionStore) ReplaceMessages(ctx context.Context, sessionID string, msgs []Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
db := s.messagesFor(sessionID)
|
||||
// Snapshot the current history first, so the delete below removes
|
||||
// exactly what was there when the call started and never touches the
|
||||
// replacement written after it.
|
||||
existing, err := db.AllDocs(ctx, sdk.DocRange{IncludeDocs: false})
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: list existing: %w", err)
|
||||
}
|
||||
doomed := make(map[string]struct{}, len(existing.Rows))
|
||||
for _, row := range existing.Rows {
|
||||
doomed[row.ID] = struct{}{}
|
||||
}
|
||||
|
||||
// Write the replacement before removing anything: an interruption here
|
||||
// leaves both sets present, which is recoverable, rather than an empty
|
||||
// session, which is not.
|
||||
//
|
||||
// A replacement message that carries an upstream ID lands on the same
|
||||
// document it already occupied, so anything just written is removed
|
||||
// from the delete set. Without this a read-modify-write caller --
|
||||
// ReplaceMessages(sess, history[10:]) -- would rewrite the survivors
|
||||
// and then delete them, emptying the session.
|
||||
for _, msg := range msgs {
|
||||
id, err := s.saveMessageLocked(ctx, sessionID, msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: %w", err)
|
||||
}
|
||||
delete(doomed, id)
|
||||
}
|
||||
|
||||
for id := range doomed {
|
||||
if _, err := db.Remove(ctx, id); err != nil && !errors.Is(err, sdk.ErrNotFound) {
|
||||
return fmt.Errorf("sessionstore: replace messages: delete %s: %w", id, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *nellSessionStore) RecentMessages(ctx context.Context, sessionID string, n int) ([]Message, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@@ -487,10 +775,15 @@ func (s *nellSessionStore) DeleteRecentMessages(ctx context.Context, sessionID s
|
||||
}
|
||||
deleted := 0
|
||||
for _, row := range rows {
|
||||
if _, err := db.Remove(ctx, row.ID); err != nil && !errors.Is(err, sdk.ErrNotFound) {
|
||||
switch _, err := db.Remove(ctx, row.ID); {
|
||||
case err == nil:
|
||||
deleted++
|
||||
case errors.Is(err, sdk.ErrNotFound):
|
||||
// Removed concurrently; it is gone either way, but it was not
|
||||
// this call that removed it, so it is not counted.
|
||||
default:
|
||||
return deleted, fmt.Errorf("sessionstore: delete message %s: %w", row.ID, err)
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
@@ -502,17 +795,12 @@ func (s *nellSessionStore) MessageCount(ctx context.Context, sessionID string) (
|
||||
}
|
||||
|
||||
func (s *nellSessionStore) messageCountLocked(ctx context.Context, sessionID string) (int, error) {
|
||||
// AllDocs already drops NellDB's internal meta documents.
|
||||
result, err := s.messagesFor(sessionID).AllDocs(ctx, sdk.DocRange{IncludeDocs: false})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sessionstore: message count: %w", err)
|
||||
}
|
||||
count := 0
|
||||
for _, row := range result.Rows {
|
||||
if !isMetaKey(row.ID) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
return len(result.Rows), nil
|
||||
}
|
||||
|
||||
// SearchMessages runs a collection-scoped text search over the session's
|
||||
@@ -537,15 +825,15 @@ func (s *nellSessionStore) SearchMessages(ctx context.Context, sessionID string,
|
||||
// request is capped at the engine ceiling; reaching it is reported as
|
||||
// Truncated rather than passed off as the end of the results.
|
||||
want := offset + limit + 1
|
||||
truncated := want > MaxSearchResults
|
||||
results, err := s.messagesFor(sessionID).TextSearch(ctx, q.Query, min(want, MaxSearchResults))
|
||||
if err != nil {
|
||||
return MessagePage{}, fmt.Errorf("sessionstore: search messages: %w", err)
|
||||
}
|
||||
// A full result set at the ceiling means matches were dropped.
|
||||
if len(results) >= MaxSearchResults {
|
||||
truncated = true
|
||||
}
|
||||
// Truncation is a property of the result set, not of the request: the
|
||||
// engine only dropped matches if it returned a full ceiling's worth
|
||||
// while we asked for at least that many. Deciding this from `want`
|
||||
// alone would flag a deep offset over a small result set as truncated.
|
||||
truncated := want >= MaxSearchResults && len(results) >= MaxSearchResults
|
||||
if offset >= len(results) {
|
||||
return MessagePage{Truncated: truncated}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,573 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// ── SQLite implementation ───────────────────────────────────────────────────
|
||||
//
|
||||
// sqliteSessionStore is the durable, transactional SessionStore described by
|
||||
// docs/architecture/messaging-and-work-intake.md (canonical session/message
|
||||
// store uses durable SQLite round trips). It coexists with nellSessionStore;
|
||||
// nothing in this file changes that implementation or the SessionStore
|
||||
// interface.
|
||||
//
|
||||
// Schema:
|
||||
//
|
||||
// - sessions holds one row per SessionContext, round-tripping every field
|
||||
// including ParentSessionID and BranchName.
|
||||
// - messages holds one row per message, keyed by the application-generated
|
||||
// canonical MessageID (session-scoped unique, not the table's rowid --
|
||||
// the integer rowid backs the FTS5 external-content index instead).
|
||||
// - messages_fts is an external-content FTS5 index over (sender, text)
|
||||
// only, kept in sync by AFTER INSERT/UPDATE/DELETE triggers on messages.
|
||||
// Because the sync is driven by triggers on the table itself rather than
|
||||
// application code, it cannot drift regardless of which Go method wrote
|
||||
// the row.
|
||||
type sqliteSessionStore struct {
|
||||
db *sql.DB
|
||||
// mu serializes every write (Save, Delete, message mutations) so the
|
||||
// strictly-increasing per-session timestamp invariant holds under
|
||||
// concurrent callers: assigning the next timestamp is a read-then-write
|
||||
// over two statements, which needs an application-level lock even
|
||||
// though SQLite itself serializes at the connection level.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
const sqliteSessionSchema = `
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
platform TEXT NOT NULL DEFAULT '',
|
||||
bot_user TEXT NOT NULL DEFAULT '',
|
||||
channel_id TEXT NOT NULL DEFAULT '',
|
||||
thread_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
parent_session_id TEXT NOT NULL DEFAULT '',
|
||||
branch_name TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL DEFAULT 0,
|
||||
last_active_at INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_channel ON sessions(platform, channel_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
message_id TEXT NOT NULL,
|
||||
session_id TEXT NOT NULL,
|
||||
source_id TEXT NOT NULL DEFAULT '',
|
||||
sender TEXT NOT NULL DEFAULT '',
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
ts INTEGER NOT NULL,
|
||||
UNIQUE(session_id, message_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_session_ts ON messages(session_id, ts);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
sender,
|
||||
text,
|
||||
content='messages',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, sender, text) VALUES('delete', old.id, old.sender, old.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, sender, text) VALUES('delete', old.id, old.sender, old.text);
|
||||
INSERT INTO messages_fts(rowid, sender, text) VALUES (new.id, new.sender, new.text);
|
||||
END;
|
||||
`
|
||||
|
||||
// OpenSQLiteSessionStore opens (creating if needed) a SQLite-backed
|
||||
// SessionStore.
|
||||
func OpenSQLiteSessionStore(path string) (SessionStore, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: mkdir: %w", err)
|
||||
}
|
||||
return openSQLiteSessionStore(path + "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)")
|
||||
}
|
||||
|
||||
// NewSQLiteSessionStoreMemory returns an in-memory SQLite SessionStore for
|
||||
// tests.
|
||||
func NewSQLiteSessionStoreMemory() (SessionStore, error) {
|
||||
return openSQLiteSessionStore(":memory:")
|
||||
}
|
||||
|
||||
func openSQLiteSessionStore(dsn string) (SessionStore, error) {
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: open: %w", err)
|
||||
}
|
||||
// A second connection to an in-memory database is a distinct, empty
|
||||
// database under modernc.org/sqlite, and even for the file-backed case
|
||||
// this store's own mutex already serializes every operation -- so a
|
||||
// pool beyond one connection buys nothing and risks the memory-mode
|
||||
// footgun. Pin it to one.
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
ctx := context.Background()
|
||||
if _, err := db.ExecContext(ctx, sqliteSessionSchema); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("sessionstore: init schema: %w", err), db.Close())
|
||||
}
|
||||
return &sqliteSessionStore{db: db}, nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) Close() error {
|
||||
return s.db.Close()
|
||||
}
|
||||
|
||||
// ── Sessions ────────────────────────────────────────────────────────────────
|
||||
|
||||
func (s *sqliteSessionStore) Save(ctx context.Context, sc SessionContext) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO sessions (session_id, platform, bot_user, channel_id, thread_id,
|
||||
title, parent_session_id, branch_name, created_at, last_active_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
platform = excluded.platform,
|
||||
bot_user = excluded.bot_user,
|
||||
channel_id = excluded.channel_id,
|
||||
thread_id = excluded.thread_id,
|
||||
title = excluded.title,
|
||||
parent_session_id = excluded.parent_session_id,
|
||||
branch_name = excluded.branch_name,
|
||||
created_at = excluded.created_at,
|
||||
last_active_at = excluded.last_active_at`,
|
||||
sc.SessionID, sc.Source.Platform, sc.Source.BotUser, sc.Source.ChannelID, sc.Source.ThreadID,
|
||||
sc.Title, sc.ParentSessionID, sc.BranchName, sc.CreatedAt.UnixMilli(), sc.LastActiveAt.UnixMilli())
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: save: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) Get(ctx context.Context, sessionID string) (*SessionContext, error) {
|
||||
row := s.db.QueryRowContext(ctx, sessionSelect+` WHERE session_id = ?`, sessionID)
|
||||
sc, err := scanSession(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: get: %w", err)
|
||||
}
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) GetByChannel(ctx context.Context, platform, channelID string) ([]SessionContext, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
sessionSelect+` WHERE platform = ? AND channel_id = ? ORDER BY created_at DESC`,
|
||||
platform, channelID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: get by channel: %w", err)
|
||||
}
|
||||
return scanSessions(rows)
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) Delete(ctx context.Context, sessionID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: delete: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM messages WHERE session_id = ?`, sessionID); err != nil {
|
||||
return fmt.Errorf("sessionstore: delete messages: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM sessions WHERE session_id = ?`, sessionID); err != nil {
|
||||
return fmt.Errorf("sessionstore: delete session: %w", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("sessionstore: delete: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) Touch(ctx context.Context, sessionID string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`UPDATE sessions SET last_active_at = ? WHERE session_id = ?`,
|
||||
time.Now().UTC().UnixMilli(), sessionID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: touch: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) List(ctx context.Context) ([]SessionContext, error) {
|
||||
rows, err := s.db.QueryContext(ctx, sessionSelect+` ORDER BY created_at DESC`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: list: %w", err)
|
||||
}
|
||||
return scanSessions(rows)
|
||||
}
|
||||
|
||||
const sessionSelect = `
|
||||
SELECT session_id, platform, bot_user, channel_id, thread_id,
|
||||
title, parent_session_id, branch_name, created_at, last_active_at
|
||||
FROM sessions`
|
||||
|
||||
// rowScanner is satisfied by both *sql.Row and *sql.Rows, letting scanSession
|
||||
// serve single-row and multi-row queries alike.
|
||||
type rowScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
func scanSession(row rowScanner) (*SessionContext, error) {
|
||||
var (
|
||||
sc SessionContext
|
||||
createdAt, lastActive int64
|
||||
)
|
||||
err := row.Scan(&sc.SessionID, &sc.Source.Platform, &sc.Source.BotUser, &sc.Source.ChannelID,
|
||||
&sc.Source.ThreadID, &sc.Title, &sc.ParentSessionID, &sc.BranchName, &createdAt, &lastActive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sc.CreatedAt = time.UnixMilli(createdAt).UTC()
|
||||
sc.LastActiveAt = time.UnixMilli(lastActive).UTC()
|
||||
return &sc, nil
|
||||
}
|
||||
|
||||
func scanSessions(rows *sql.Rows) ([]SessionContext, error) {
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []SessionContext
|
||||
for rows.Next() {
|
||||
sc, err := scanSession(rows)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan session: %w", err)
|
||||
}
|
||||
out = append(out, *sc)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan sessions: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ── Messages ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// execer is satisfied by both *sql.DB and *sql.Tx, so message-writing
|
||||
// helpers work identically for a single-statement call (SaveMessage) and a
|
||||
// multi-statement transaction (SaveMessages, ReplaceMessages).
|
||||
type execer interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) SaveMessage(ctx context.Context, sessionID string, msg Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, err := saveMessage(ctx, s.db, sessionID, msg)
|
||||
return err
|
||||
}
|
||||
|
||||
// saveMessage persists one message and reports the canonical ID it was
|
||||
// stored under (or was already stored under, for a redelivered message).
|
||||
//
|
||||
// A message carrying a canonical ID keeps it -- re-saving history read back
|
||||
// must not mint a new identity. An existing record with that ID is left
|
||||
// untouched: stored messages are immutable, so a redelivered edit is a
|
||||
// no-op rather than an overwrite.
|
||||
func saveMessage(ctx context.Context, ex execer, sessionID string, msg Message) (string, error) {
|
||||
id := msg.MessageID
|
||||
if id == "" {
|
||||
id = newMessageID(sessionID, msg.SourceID)
|
||||
}
|
||||
|
||||
var exists int
|
||||
err := ex.QueryRowContext(ctx,
|
||||
`SELECT 1 FROM messages WHERE session_id = ? AND message_id = ?`, sessionID, id).Scan(&exists)
|
||||
switch {
|
||||
case err == nil:
|
||||
return id, nil
|
||||
case !errors.Is(err, sql.ErrNoRows):
|
||||
return "", fmt.Errorf("sessionstore: read existing message: %w", err)
|
||||
}
|
||||
|
||||
at, err := nextTimestampSQL(ctx, ex, sessionID, stamp(msg))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_, err = ex.ExecContext(ctx, `
|
||||
INSERT INTO messages (message_id, session_id, source_id, sender, text, ts)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
id, sessionID, msg.SourceID, msg.From, msg.Text, at.UnixMilli())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("sessionstore: save message: %w", err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// nextTimestampSQL picks the timestamp a message is persisted under, kept
|
||||
// strictly increasing within a session so history stays an append-only
|
||||
// linear sequence. Channels report time at wildly different resolutions --
|
||||
// a Telegram message carries whole seconds while a locally generated reply
|
||||
// carries milliseconds -- so the sender's clock is honoured whenever it is
|
||||
// already ahead of the session's newest, and otherwise advanced by the
|
||||
// smallest representable step.
|
||||
func nextTimestampSQL(ctx context.Context, ex execer, sessionID string, want time.Time) (time.Time, error) {
|
||||
var last sql.NullInt64
|
||||
err := ex.QueryRowContext(ctx,
|
||||
`SELECT MAX(ts) FROM messages WHERE session_id = ?`, sessionID).Scan(&last)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("sessionstore: read newest message: %w", err)
|
||||
}
|
||||
ms := want.UnixMilli()
|
||||
if last.Valid && ms <= last.Int64 {
|
||||
ms = last.Int64 + 1
|
||||
}
|
||||
return time.UnixMilli(ms).UTC(), nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) SaveMessages(ctx context.Context, sessionID string, msgs []Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: save messages: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
// Each message gets its own time-ordered ID, so a bulk save appends to
|
||||
// whatever the session already holds instead of overwriting from zero.
|
||||
for _, msg := range msgs {
|
||||
if _, err := saveMessage(ctx, tx, sessionID, msg); err != nil {
|
||||
return fmt.Errorf("sessionstore: save messages: %w", err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("sessionstore: save messages: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReplaceMessages makes msgs the session's entire history, atomically: the
|
||||
// replacement is written and the previous history removed inside a single
|
||||
// transaction, so a caller either sees the old history or the new one, never
|
||||
// an in-between or empty state.
|
||||
//
|
||||
// A replacement message that carries a canonical ID lands on the same row it
|
||||
// already occupied (saveMessage's immutability no-op), so a read-modify-write
|
||||
// caller -- ReplaceMessages(sess, history[:2]) -- keeps those rows rather than
|
||||
// deleting them: they are excluded from the survivor-complement delete below.
|
||||
func (s *sqliteSessionStore) ReplaceMessages(ctx context.Context, sessionID string, msgs []Message) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: begin: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
survivors := make([]string, 0, len(msgs))
|
||||
for _, msg := range msgs {
|
||||
id, err := saveMessage(ctx, tx, sessionID, msg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: %w", err)
|
||||
}
|
||||
survivors = append(survivors, id)
|
||||
}
|
||||
|
||||
query := `DELETE FROM messages WHERE session_id = ?`
|
||||
args := []any{sessionID}
|
||||
if len(survivors) > 0 {
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(survivors)), ",")
|
||||
query += ` AND message_id NOT IN (` + placeholders + `)`
|
||||
for _, id := range survivors {
|
||||
args = append(args, id)
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: delete old: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("sessionstore: replace messages: commit: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) RecentMessages(ctx context.Context, sessionID string, n int) ([]Message, error) {
|
||||
if n <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT message_id, source_id, sender, text, ts FROM (
|
||||
SELECT message_id, source_id, sender, text, ts FROM messages
|
||||
WHERE session_id = ? ORDER BY ts DESC LIMIT ?
|
||||
) ORDER BY ts ASC`, sessionID, n)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: recent messages: %w", err)
|
||||
}
|
||||
return scanMessages(rows)
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) DeleteRecentMessages(ctx context.Context, sessionID string, n int) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if n <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
res, err := s.db.ExecContext(ctx, `
|
||||
DELETE FROM messages WHERE id IN (
|
||||
SELECT id FROM messages WHERE session_id = ? ORDER BY ts DESC LIMIT ?
|
||||
)`, sessionID, n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sessionstore: delete recent messages: %w", err)
|
||||
}
|
||||
deleted, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sessionstore: delete recent messages: rows affected: %w", err)
|
||||
}
|
||||
return int(deleted), nil
|
||||
}
|
||||
|
||||
func (s *sqliteSessionStore) MessageCount(ctx context.Context, sessionID string) (int, error) {
|
||||
var n int
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM messages WHERE session_id = ?`, sessionID).Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sessionstore: message count: %w", err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func scanMessages(rows *sql.Rows) ([]Message, error) {
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []Message
|
||||
for rows.Next() {
|
||||
var (
|
||||
msg Message
|
||||
ts int64
|
||||
)
|
||||
if err := rows.Scan(&msg.MessageID, &msg.SourceID, &msg.From, &msg.Text, &ts); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan message: %w", err)
|
||||
}
|
||||
msg.At = time.UnixMilli(ts).UTC()
|
||||
out = append(out, msg)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan messages: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SearchMessages runs an FTS5 search over the session's entire message
|
||||
// history, matching message text and sender only -- ts and other metadata
|
||||
// are not part of the indexed content, so a query like "2026" cannot match
|
||||
// every message via its timestamp. An empty or whitespace-only query
|
||||
// returns an empty page without touching the database.
|
||||
//
|
||||
// SQLite gives an exact total match count, so paging here has no analogue
|
||||
// to NellDB's MaxTextSearchLimit ceiling: MessagePage.Truncated is always
|
||||
// false.
|
||||
func (s *sqliteSessionStore) SearchMessages(ctx context.Context, sessionID string, q MessageQuery) (MessagePage, error) {
|
||||
if strings.TrimSpace(q.Query) == "" {
|
||||
return MessagePage{}, nil
|
||||
}
|
||||
limit := q.Limit
|
||||
if limit <= 0 {
|
||||
limit = DefaultMessagePageSize
|
||||
}
|
||||
limit = min(limit, MaxMessagePageSize)
|
||||
offset := max(q.Offset, 0)
|
||||
|
||||
match := ftsMatchQuery(q.Query)
|
||||
|
||||
var total int
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM messages_fts
|
||||
JOIN messages m ON m.id = messages_fts.rowid
|
||||
WHERE messages_fts MATCH ? AND m.session_id = ?`, match, sessionID).Scan(&total)
|
||||
if err != nil {
|
||||
return MessagePage{}, fmt.Errorf("sessionstore: search messages: count: %w", err)
|
||||
}
|
||||
if offset >= total {
|
||||
return MessagePage{}, nil
|
||||
}
|
||||
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT m.message_id, m.source_id, m.sender, m.text, m.ts
|
||||
FROM messages_fts
|
||||
JOIN messages m ON m.id = messages_fts.rowid
|
||||
WHERE messages_fts MATCH ? AND m.session_id = ?
|
||||
ORDER BY bm25(messages_fts) ASC
|
||||
LIMIT ? OFFSET ?`, match, sessionID, limit, offset)
|
||||
if err != nil {
|
||||
return MessagePage{}, fmt.Errorf("sessionstore: search messages: %w", err)
|
||||
}
|
||||
msgs, err := scanSearchMessages(rows)
|
||||
if err != nil {
|
||||
return MessagePage{}, err
|
||||
}
|
||||
|
||||
next := offset + len(msgs)
|
||||
return MessagePage{
|
||||
Messages: msgs,
|
||||
NextOffset: next,
|
||||
HasMore: next < total,
|
||||
Truncated: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// scanSearchMessages scans rows shaped (message_id, source_id, sender, text,
|
||||
// ts) -- the same column order SearchMessages selects, distinct from
|
||||
// scanMessages' column order because the FTS join names "sender" rather
|
||||
// than the "from" convention the rest of the file uses.
|
||||
func scanSearchMessages(rows *sql.Rows) ([]Message, error) {
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []Message
|
||||
for rows.Next() {
|
||||
var (
|
||||
msg Message
|
||||
ts int64
|
||||
)
|
||||
if err := rows.Scan(&msg.MessageID, &msg.SourceID, &msg.From, &msg.Text, &ts); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan search result: %w", err)
|
||||
}
|
||||
msg.At = time.UnixMilli(ts).UTC()
|
||||
out = append(out, msg)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("sessionstore: scan search results: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ftsMatchQuery turns free text into an FTS5 MATCH expression requiring
|
||||
// every term to be present. Each term is wrapped as a quoted phrase --
|
||||
// doubling any embedded quote to escape it -- so user input can never be
|
||||
// interpreted as FTS5 query syntax (column filters, NEAR, boolean
|
||||
// operators); every token is a literal string to match, joined with an
|
||||
// explicit AND.
|
||||
func ftsMatchQuery(query string) string {
|
||||
fields := strings.Fields(query)
|
||||
terms := make([]string, 0, len(fields))
|
||||
for _, f := range fields {
|
||||
escaped := strings.ReplaceAll(f, `"`, `""`)
|
||||
terms = append(terms, `"`+escaped+`"`)
|
||||
}
|
||||
return strings.Join(terms, " AND ")
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package gateway
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestSessionStoreConformanceSQLite runs the shared SessionStore contract
|
||||
// against the SQLite implementation. It is the migration's actual gate:
|
||||
// both backends must satisfy identical behaviour, so this suite and its
|
||||
// NellDB counterpart must stay in lockstep.
|
||||
func TestSessionStoreConformanceSQLite(t *testing.T) {
|
||||
runSessionStoreSuite(t, func(t *testing.T) SessionStore {
|
||||
s, err := NewSQLiteSessionStoreMemory()
|
||||
if err != nil {
|
||||
t.Fatalf("NewSQLiteSessionStoreMemory: %v", err)
|
||||
}
|
||||
return s
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,762 @@
|
||||
package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newTestSQLiteStore(t *testing.T) SessionStore {
|
||||
t.Helper()
|
||||
st, err := NewSQLiteSessionStoreMemory()
|
||||
if err != nil {
|
||||
t.Fatalf("NewSQLiteSessionStoreMemory: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
func sqliteBase() time.Time { return time.Date(2026, 8, 3, 12, 0, 0, 0, time.UTC) }
|
||||
|
||||
// TestSQLiteSessionStore_DeleteIsAtomic pins invariant 4: Delete removes the
|
||||
// session and its messages together.
|
||||
func TestSQLiteSessionStore_DeleteIsAtomic(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
sc := SessionContext{
|
||||
SessionID: "sess-del", Source: SessionSource{Platform: "web", ChannelID: "c"},
|
||||
CreatedAt: sqliteBase(), LastActiveAt: sqliteBase(),
|
||||
}
|
||||
if err := st.Save(ctx, sc); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
if err := st.SaveMessage(ctx, "sess-del", Message{From: "u", Text: "hi", At: sqliteBase()}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
|
||||
if err := st.Delete(ctx, "sess-del"); err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
|
||||
got, err := st.Get(ctx, "sess-del")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if got != nil {
|
||||
t.Error("expected session to be gone")
|
||||
}
|
||||
count, err := st.MessageCount(ctx, "sess-del")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Errorf("expected messages to be gone, got count %d", count)
|
||||
}
|
||||
|
||||
// Deleting a non-existent session is a no-op.
|
||||
if err := st.Delete(ctx, "nonexistent"); err != nil {
|
||||
t.Errorf("Delete nonexistent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_SearchPagingAndTruncated pins invariant 6: Limit is
|
||||
// a page size (not a cap), paging is honoured via Offset/NextOffset/HasMore,
|
||||
// and Truncated is always false (SQLite has a real total count).
|
||||
func TestSQLiteSessionStore_SearchPagingAndTruncated(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
const total = 30
|
||||
for i := range total {
|
||||
m := Message{From: "u", Text: "needle common", At: sqliteBase().Add(time.Duration(i) * time.Second)}
|
||||
if err := st.SaveMessage(ctx, "sess", m); err != nil {
|
||||
t.Fatalf("SaveMessage %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
offset := 0
|
||||
pages := 0
|
||||
for {
|
||||
page, err := st.SearchMessages(ctx, "sess", MessageQuery{Query: "needle", Limit: 7, Offset: offset})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages offset=%d: %v", offset, err)
|
||||
}
|
||||
if page.Truncated {
|
||||
t.Errorf("expected Truncated=false for SQLite backend")
|
||||
}
|
||||
for _, m := range page.Messages {
|
||||
if seen[m.MessageID] {
|
||||
t.Errorf("duplicate message across pages: %s", m.MessageID)
|
||||
}
|
||||
seen[m.MessageID] = true
|
||||
}
|
||||
pages++
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
offset = page.NextOffset
|
||||
if pages > total {
|
||||
t.Fatal("paging did not terminate")
|
||||
}
|
||||
}
|
||||
if len(seen) != total {
|
||||
t.Errorf("expected to see all %d matches across pages, got %d", total, len(seen))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteSessionStore_ErrorPaths(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
t.Run("Save after close", func(t *testing.T) {
|
||||
if err := st.Save(ctx, SessionContext{SessionID: "x"}); err == nil {
|
||||
t.Error("expected error saving to a closed store")
|
||||
}
|
||||
})
|
||||
t.Run("Get after close", func(t *testing.T) {
|
||||
if _, err := st.Get(ctx, "x"); err == nil {
|
||||
t.Error("expected error getting from a closed store")
|
||||
}
|
||||
})
|
||||
t.Run("SaveMessage after close", func(t *testing.T) {
|
||||
if err := st.SaveMessage(ctx, "sess", Message{From: "u", Text: "hi"}); err == nil {
|
||||
t.Error("expected error saving message to a closed store")
|
||||
}
|
||||
})
|
||||
t.Run("SearchMessages after close", func(t *testing.T) {
|
||||
if _, err := st.SearchMessages(ctx, "sess", MessageQuery{Query: "hi"}); err == nil {
|
||||
t.Error("expected error searching a closed store")
|
||||
}
|
||||
})
|
||||
t.Run("Delete after close", func(t *testing.T) {
|
||||
if err := st.Delete(ctx, "sess"); err == nil {
|
||||
t.Error("expected error deleting from a closed store")
|
||||
}
|
||||
})
|
||||
t.Run("ReplaceMessages after close", func(t *testing.T) {
|
||||
if err := st.ReplaceMessages(ctx, "sess", nil); err == nil {
|
||||
t.Error("expected error replacing messages on a closed store")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOpenSQLiteSessionStore_CreatesFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := dir + "/sub/dir/sessions.db"
|
||||
|
||||
st, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore: %v", err)
|
||||
}
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
ctx := context.Background()
|
||||
sc := SessionContext{SessionID: "s1", Source: SessionSource{Platform: "web", ChannelID: "c"}, CreatedAt: sqliteBase(), LastActiveAt: sqliteBase()}
|
||||
if err := st.Save(ctx, sc); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
got, err := st.Get(ctx, "s1")
|
||||
if err != nil {
|
||||
t.Fatalf("Get: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected session to persist to the created file")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Adversarial file-backed coverage (bead 8u8.1) ───────────────────────────
|
||||
//
|
||||
// These tests are deliberately SQLite-only. They prove two properties the
|
||||
// shared 19-group conformance suite cannot: file-backed durability across
|
||||
// close+reopen (with the trigger-maintained FTS index intact and a clean
|
||||
// PRAGMA quick_check), and transaction rollback under injected
|
||||
// mid-operation failures. They reach into sqliteSessionStore internals -- the
|
||||
// raw *sql.DB -- to install deterministic abort triggers and run PRAGMA
|
||||
// checks, which the SessionStore interface does not expose.
|
||||
|
||||
// sqliteStoreDB exposes the raw handle behind a store for SQLite-specific
|
||||
// tests. The store pins its pool to one connection and serializes writes
|
||||
// with its own mutex, so a test touching this handle between operations is
|
||||
// safe and deterministic.
|
||||
func sqliteStoreDB(t *testing.T, st SessionStore) *sql.DB {
|
||||
t.Helper()
|
||||
s, ok := st.(*sqliteSessionStore)
|
||||
if !ok {
|
||||
t.Fatalf("store is %T, want *sqliteSessionStore", st)
|
||||
}
|
||||
return s.db
|
||||
}
|
||||
|
||||
// installAbortTrigger deterministically fails the statement that matches
|
||||
// clause by raising ABORT from a BEFORE trigger. Because the trigger fires
|
||||
// mid-transaction (after earlier statements in the same transaction have
|
||||
// already run), a correct store must roll the whole transaction back for the
|
||||
// abort to be invisible to later reads.
|
||||
func installAbortTrigger(t *testing.T, st SessionStore, clause string) {
|
||||
t.Helper()
|
||||
_, err := sqliteStoreDB(t, st).ExecContext(context.Background(),
|
||||
`CREATE TRIGGER test_forced_abort `+clause+` BEGIN
|
||||
SELECT RAISE(ABORT, 'forced test abort');
|
||||
END`)
|
||||
if err != nil {
|
||||
t.Fatalf("install abort trigger: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_CloseReopenDurability proves that a file-backed
|
||||
// store persists SessionContext fields and canonical message identity
|
||||
// (MessageID, SourceID, sender, text, timestamp) across a close+reopen
|
||||
// cycle, that the trigger-maintained FTS index survives with it, and that
|
||||
// PRAGMA quick_check reports the file intact.
|
||||
func TestSQLiteSessionStore_CloseReopenDurability(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "sessions.db")
|
||||
|
||||
st, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSQLiteSessionStore: %v", err)
|
||||
}
|
||||
|
||||
wantSC := SessionContext{
|
||||
SessionID: "sess-durable",
|
||||
Source: SessionSource{
|
||||
Platform: "telegram", BotUser: "archie", ChannelID: "chat-42", ThreadID: "topic-7",
|
||||
},
|
||||
Title: "durable title",
|
||||
ParentSessionID: "sess-parent",
|
||||
BranchName: "feature/x",
|
||||
CreatedAt: sqliteBase(),
|
||||
LastActiveAt: sqliteBase().Add(time.Minute),
|
||||
}
|
||||
if err := st.Save(ctx, wantSC); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
|
||||
wantMsgs := []Message{
|
||||
{MessageID: "canonical-1", SourceID: "tg-1001", From: "alice", Text: "first durable message", At: sqliteBase().Add(time.Second)},
|
||||
{MessageID: "canonical-2", SourceID: "tg-1002", From: "bob", Text: "second durable message", At: sqliteBase().Add(2 * time.Second)},
|
||||
}
|
||||
for _, m := range wantMsgs {
|
||||
if err := st.SaveMessage(ctx, "sess-durable", m); err != nil {
|
||||
t.Fatalf("SaveMessage %q: %v", m.MessageID, err)
|
||||
}
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
st2, err := OpenSQLiteSessionStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen OpenSQLiteSessionStore: %v", err)
|
||||
}
|
||||
defer func() { _ = st2.Close() }()
|
||||
|
||||
got, err := st2.Get(ctx, "sess-durable")
|
||||
if err != nil {
|
||||
t.Fatalf("Get after reopen: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected session to survive close+reopen")
|
||||
}
|
||||
if *got != wantSC {
|
||||
t.Errorf("session after reopen:\n got %+v\nwant %+v", *got, wantSC)
|
||||
}
|
||||
|
||||
hist, err := st2.RecentMessages(ctx, "sess-durable", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages after reopen: %v", err)
|
||||
}
|
||||
if len(hist) != len(wantMsgs) {
|
||||
t.Fatalf("history after reopen: got %d messages, want %d", len(hist), len(wantMsgs))
|
||||
}
|
||||
for i, m := range hist {
|
||||
want := wantMsgs[i]
|
||||
if m.MessageID != want.MessageID || m.SourceID != want.SourceID ||
|
||||
m.From != want.From || m.Text != want.Text || !m.At.Equal(want.At) {
|
||||
t.Errorf("message %d after reopen:\n got %+v\nwant %+v", i, m, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The FTS index is trigger-maintained, so it must survive reopen too.
|
||||
page, err := st2.SearchMessages(ctx, "sess-durable", MessageQuery{Query: "durable"})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages after reopen: %v", err)
|
||||
}
|
||||
if len(page.Messages) != len(wantMsgs) {
|
||||
t.Errorf("search after reopen: got %d matches, want %d", len(page.Messages), len(wantMsgs))
|
||||
}
|
||||
|
||||
var check string
|
||||
if err := sqliteStoreDB(t, st2).QueryRowContext(ctx, "PRAGMA quick_check").Scan(&check); err != nil {
|
||||
t.Fatalf("PRAGMA quick_check: %v", err)
|
||||
}
|
||||
if check != "ok" {
|
||||
t.Errorf("PRAGMA quick_check = %q, want ok", check)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_DeleteAbortRollsBack pins that Delete rolls its
|
||||
// transaction back when a statement fails mid-operation: the messages DELETE
|
||||
// has already run inside the transaction when the sessions DELETE trips the
|
||||
// abort trigger, so the rollback must restore the session, its history, its
|
||||
// count, and its search results.
|
||||
func TestSQLiteSessionStore_DeleteAbortRollsBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
sc := SessionContext{
|
||||
SessionID: "sess-del-abort",
|
||||
Source: SessionSource{Platform: "web", ChannelID: "c"},
|
||||
Title: "keep me",
|
||||
CreatedAt: sqliteBase(), LastActiveAt: sqliteBase(),
|
||||
}
|
||||
if err := st.Save(ctx, sc); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
seed := []Message{
|
||||
{From: "alice", Text: "alpha", At: sqliteBase()},
|
||||
{From: "bob", Text: "beta", At: sqliteBase().Add(time.Second)},
|
||||
{From: "carol", Text: "gamma", At: sqliteBase().Add(2 * time.Second)},
|
||||
}
|
||||
for _, m := range seed {
|
||||
if err := st.SaveMessage(ctx, "sess-del-abort", m); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
installAbortTrigger(t, st, `BEFORE DELETE ON sessions WHEN old.session_id = 'sess-del-abort'`)
|
||||
|
||||
if err := st.Delete(ctx, "sess-del-abort"); err == nil {
|
||||
t.Fatal("expected Delete to fail from the forced abort trigger")
|
||||
}
|
||||
|
||||
got, err := st.Get(ctx, "sess-del-abort")
|
||||
if err != nil {
|
||||
t.Fatalf("Get after failed Delete: %v", err)
|
||||
}
|
||||
if got == nil || *got != sc {
|
||||
t.Errorf("session after failed Delete: got %+v, want %+v", got, sc)
|
||||
}
|
||||
|
||||
hist, err := st.RecentMessages(ctx, "sess-del-abort", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages after failed Delete: %v", err)
|
||||
}
|
||||
if len(hist) != len(seed) {
|
||||
t.Fatalf("history after failed Delete: got %d messages, want %d", len(hist), len(seed))
|
||||
}
|
||||
for i, m := range hist {
|
||||
if m.Text != seed[i].Text || m.From != seed[i].From {
|
||||
t.Errorf("message %d after failed Delete: got %+v, want text %q from %q", i, m, seed[i].Text, seed[i].From)
|
||||
}
|
||||
}
|
||||
|
||||
count, err := st.MessageCount(ctx, "sess-del-abort")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount after failed Delete: %v", err)
|
||||
}
|
||||
if count != len(seed) {
|
||||
t.Errorf("count after failed Delete: got %d, want %d", count, len(seed))
|
||||
}
|
||||
|
||||
// The FTS index must match the rolled-back history.
|
||||
page, err := st.SearchMessages(ctx, "sess-del-abort", MessageQuery{Query: "beta"})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages after failed Delete: %v", err)
|
||||
}
|
||||
if len(page.Messages) != 1 {
|
||||
t.Errorf("search after failed Delete: got %d matches, want 1", len(page.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_ReplaceMessagesAbortRollsBack pins that
|
||||
// ReplaceMessages rolls its transaction back when the delete-old step fails
|
||||
// mid-operation: the replacement inserts have already landed in the
|
||||
// transaction when the abort fires, so the rollback must remove the
|
||||
// replacement and restore the original history, count, and search results.
|
||||
func TestSQLiteSessionStore_ReplaceMessagesAbortRollsBack(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
seed := []Message{
|
||||
{MessageID: "m-0", SourceID: "s-0", From: "alice", Text: "alpha", At: sqliteBase()},
|
||||
{MessageID: "m-1", SourceID: "s-1", From: "bob", Text: "beta", At: sqliteBase().Add(time.Second)},
|
||||
{MessageID: "m-2", SourceID: "s-2", From: "carol", Text: "gamma", At: sqliteBase().Add(2 * time.Second)},
|
||||
}
|
||||
if err := st.SaveMessages(ctx, "sess-replace-abort", seed); err != nil {
|
||||
t.Fatalf("SaveMessages: %v", err)
|
||||
}
|
||||
|
||||
// Guard an original row so the abort fires only after the replacement
|
||||
// inserts have landed inside the transaction.
|
||||
installAbortTrigger(t, st, `BEFORE DELETE ON messages WHEN old.message_id = 'm-1'`)
|
||||
|
||||
replacement := []Message{{From: "summarizer", Text: "merged summary", At: sqliteBase().Add(time.Hour)}}
|
||||
if err := st.ReplaceMessages(ctx, "sess-replace-abort", replacement); err == nil {
|
||||
t.Fatal("expected ReplaceMessages to fail from the forced abort trigger")
|
||||
}
|
||||
|
||||
hist, err := st.RecentMessages(ctx, "sess-replace-abort", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages after failed ReplaceMessages: %v", err)
|
||||
}
|
||||
if len(hist) != len(seed) {
|
||||
t.Fatalf("history after failed ReplaceMessages: got %d messages, want %d", len(hist), len(seed))
|
||||
}
|
||||
for i, m := range hist {
|
||||
if m.MessageID != seed[i].MessageID || m.Text != seed[i].Text || m.From != seed[i].From {
|
||||
t.Errorf("message %d after failed ReplaceMessages: got %+v, want %+v", i, m, seed[i])
|
||||
}
|
||||
}
|
||||
|
||||
count, err := st.MessageCount(ctx, "sess-replace-abort")
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount after failed ReplaceMessages: %v", err)
|
||||
}
|
||||
if count != len(seed) {
|
||||
t.Errorf("count after failed ReplaceMessages: got %d, want %d", count, len(seed))
|
||||
}
|
||||
|
||||
page, err := st.SearchMessages(ctx, "sess-replace-abort", MessageQuery{Query: "beta"})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages after failed ReplaceMessages: %v", err)
|
||||
}
|
||||
if len(page.Messages) != 1 {
|
||||
t.Errorf("search 'beta' after failed ReplaceMessages: got %d matches, want 1", len(page.Messages))
|
||||
}
|
||||
page, err = st.SearchMessages(ctx, "sess-replace-abort", MessageQuery{Query: "summary"})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages after failed ReplaceMessages: %v", err)
|
||||
}
|
||||
if len(page.Messages) != 0 {
|
||||
t.Errorf("search 'summary' after failed ReplaceMessages: got %d matches, want 0 (replacement must be rolled back)", len(page.Messages))
|
||||
}
|
||||
}
|
||||
|
||||
// ── FTS lifecycle and literal-query semantics (bead 8u8.4) ──────────────────
|
||||
//
|
||||
// The shared 19-group conformance suite (sessionstore_conformance_test.go)
|
||||
// proves behaviour through the SessionStore interface, but two properties are
|
||||
// SQLite-specific and cannot be expressed there: the trigger-maintained FTS5
|
||||
// index must stay consistent as history mutates, and free-text queries must
|
||||
// survive hostile input. These tests are SQLite-only and assert exact
|
||||
// results -- not merely the absence of panics.
|
||||
|
||||
// sameTexts compares two text slices as sets: same length, same members,
|
||||
// order-insensitive. FTS pages are ranked by bm25, so order is a ranking
|
||||
// detail rather than part of the match contract.
|
||||
func sameTexts(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]int, len(a))
|
||||
for _, s := range a {
|
||||
seen[s]++
|
||||
}
|
||||
for _, s := range b {
|
||||
seen[s]--
|
||||
if seen[s] < 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ftsIntegrityErr runs the documented FTS5 external-content integrity
|
||||
// check (`INSERT INTO ...(table, rank) VALUES('integrity-check', 1)`) against
|
||||
// the store's raw handle and returns any error it reports. A consistent
|
||||
// index reports no error; a stale entry -- a content row deleted without the
|
||||
// AFTER DELETE trigger firing -- reports "database disk image is malformed".
|
||||
// Search/count joins can never surface such an entry because they only see
|
||||
// index rows whose content row still exists, so this is the one check that
|
||||
// detects an orphaned index.
|
||||
func ftsIntegrityErr(t *testing.T, st SessionStore) error {
|
||||
t.Helper()
|
||||
_, err := sqliteStoreDB(t, st).ExecContext(context.Background(),
|
||||
`INSERT INTO messages_fts(messages_fts, rank) VALUES('integrity-check', 1)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_FTSStaysCorrectThroughMutations pins that the FTS5
|
||||
// index follows every message mutation: DeleteRecentMessages removes the
|
||||
// newest documents, ReplaceMessages swaps the whole document set, and Delete
|
||||
// clears the session -- with search results and counts tracking the surviving
|
||||
// history and never leaking across sessions.
|
||||
func TestSQLiteSessionStore_FTSStaysCorrectThroughMutations(t *testing.T) {
|
||||
// Sessions "a" and "b" share the term "needle", so any cross-session
|
||||
// leak in search or count is caught; "marker" lives only in "b".
|
||||
seedA := []string{"needle alpha", "filler one", "needle beta", "filler two"}
|
||||
seedB := []string{"needle gamma", "distinct marker"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(ctx context.Context, t *testing.T, st SessionStore) error
|
||||
wantA []string // session a history after the mutation
|
||||
wantANeedle []string // session a search "needle"
|
||||
wantACount int
|
||||
wantAMarker []string // session a search "marker" (only in b)
|
||||
wantBNeedle []string // session b search "needle" (must stay intact)
|
||||
wantBCount int
|
||||
}{
|
||||
{
|
||||
name: "DeleteRecentMessages prunes the index newest-first",
|
||||
mutate: func(ctx context.Context, t *testing.T, st SessionStore) error {
|
||||
deleted, err := st.DeleteRecentMessages(ctx, "a", 2)
|
||||
if err == nil && deleted != 2 {
|
||||
t.Errorf("deleted %d messages, want 2", deleted)
|
||||
}
|
||||
return err
|
||||
},
|
||||
wantA: []string{"needle alpha", "filler one"},
|
||||
wantANeedle: []string{"needle alpha"},
|
||||
wantACount: 2,
|
||||
wantAMarker: nil,
|
||||
wantBNeedle: []string{"needle gamma"},
|
||||
wantBCount: 2,
|
||||
},
|
||||
{
|
||||
name: "ReplaceMessages swaps the document set",
|
||||
mutate: func(ctx context.Context, t *testing.T, st SessionStore) error {
|
||||
return st.ReplaceMessages(ctx, "a", []Message{{
|
||||
From: "summarizer", Text: "merged needle summary", At: sqliteBase().Add(time.Hour),
|
||||
}})
|
||||
},
|
||||
wantA: []string{"merged needle summary"},
|
||||
wantANeedle: []string{"merged needle summary"},
|
||||
wantACount: 1,
|
||||
wantAMarker: nil,
|
||||
wantBNeedle: []string{"needle gamma"},
|
||||
wantBCount: 2,
|
||||
},
|
||||
{
|
||||
name: "Delete clears the session's index",
|
||||
mutate: func(ctx context.Context, t *testing.T, st SessionStore) error {
|
||||
return st.Delete(ctx, "a")
|
||||
},
|
||||
wantA: nil,
|
||||
wantANeedle: nil,
|
||||
wantACount: 0,
|
||||
wantAMarker: nil,
|
||||
wantBNeedle: []string{"needle gamma"},
|
||||
wantBCount: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Subtest-local context: mutation closures receive it as a
|
||||
// parameter instead of capturing the outer function's.
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
for i, text := range seedA {
|
||||
if err := st.SaveMessage(ctx, "a", Message{From: "u", Text: text, At: sqliteBase().Add(time.Duration(i) * time.Second)}); err != nil {
|
||||
t.Fatalf("SaveMessage a %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
for i, text := range seedB {
|
||||
if err := st.SaveMessage(ctx, "b", Message{From: "u", Text: text, At: sqliteBase().Add(time.Duration(i) * time.Second)}); err != nil {
|
||||
t.Fatalf("SaveMessage b %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tc.mutate(ctx, t, st); err != nil {
|
||||
t.Fatalf("mutate: %v", err)
|
||||
}
|
||||
|
||||
// The index must be internally consistent with the surviving
|
||||
// history, not merely join-visible: an orphaned entry would be
|
||||
// invisible to the search/count joins below, so check the index
|
||||
// itself after every mutation.
|
||||
if err := ftsIntegrityErr(t, st); err != nil {
|
||||
t.Fatalf("FTS5 integrity check after %q: %v", tc.name, err)
|
||||
}
|
||||
|
||||
hist, err := st.RecentMessages(ctx, "a", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages a: %v", err)
|
||||
}
|
||||
if !equal(texts(hist), tc.wantA) {
|
||||
t.Errorf("session a history = %v, want %v", texts(hist), tc.wantA)
|
||||
}
|
||||
|
||||
assertSearch := func(session, term string, want []string) {
|
||||
t.Helper()
|
||||
page, err := st.SearchMessages(ctx, session, MessageQuery{Query: term, Limit: 100})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages(%s, %q): %v", session, term, err)
|
||||
}
|
||||
if !sameTexts(texts(page.Messages), want) {
|
||||
t.Errorf("search %s %q = %v, want %v", session, term, texts(page.Messages), want)
|
||||
}
|
||||
if page.Truncated || page.HasMore {
|
||||
t.Errorf("search %s %q: Truncated=%v HasMore=%v, want false/false", session, term, page.Truncated, page.HasMore)
|
||||
}
|
||||
}
|
||||
|
||||
assertSearch("a", "needle", tc.wantANeedle)
|
||||
assertSearch("a", "marker", tc.wantAMarker)
|
||||
assertSearch("b", "needle", tc.wantBNeedle)
|
||||
|
||||
for _, sc := range []struct {
|
||||
session string
|
||||
want int
|
||||
}{
|
||||
{"a", tc.wantACount}, {"b", tc.wantBCount},
|
||||
} {
|
||||
count, err := st.MessageCount(ctx, sc.session)
|
||||
if err != nil {
|
||||
t.Fatalf("MessageCount %s: %v", sc.session, err)
|
||||
}
|
||||
if count != sc.want {
|
||||
t.Errorf("MessageCount %s = %d, want %d", sc.session, count, sc.want)
|
||||
}
|
||||
}
|
||||
|
||||
histB, err := st.RecentMessages(ctx, "b", 100)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages b: %v", err)
|
||||
}
|
||||
if !equal(texts(histB), seedB) {
|
||||
t.Errorf("session b history = %v, want untouched %v", texts(histB), seedB)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_FTSIntegrityCheckDetectsOrphanedIndex proves the
|
||||
// integrity check used by the lifecycle test is load-bearing: with the AFTER
|
||||
// DELETE trigger temporarily dropped, a store mutation leaves a stale
|
||||
// messages_fts entry that the check must report, while the same mutation with
|
||||
// the trigger intact leaves a consistent index.
|
||||
func TestSQLiteSessionStore_FTSIntegrityCheckDetectsOrphanedIndex(t *testing.T) {
|
||||
t.Run("stale index entry fails the check", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
for i, text := range []string{"needle one", "needle two"} {
|
||||
if err := st.SaveMessage(ctx, "sess", Message{From: "u", Text: text, At: sqliteBase().Add(time.Duration(i) * time.Second)}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Bypass the AFTER DELETE trigger so the index keeps a row for the
|
||||
// deleted message.
|
||||
if _, err := sqliteStoreDB(t, st).ExecContext(ctx, `DROP TRIGGER messages_ad`); err != nil {
|
||||
t.Fatalf("drop delete trigger: %v", err)
|
||||
}
|
||||
if _, err := st.DeleteRecentMessages(ctx, "sess", 1); err != nil {
|
||||
t.Fatalf("DeleteRecentMessages: %v", err)
|
||||
}
|
||||
|
||||
if err := ftsIntegrityErr(t, st); err == nil {
|
||||
t.Fatal("FTS5 integrity check passed despite a stale index entry (delete trigger bypassed)")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trigger-maintained delete keeps the index consistent", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
for i, text := range []string{"needle one", "needle two"} {
|
||||
if err := st.SaveMessage(ctx, "sess", Message{From: "u", Text: text, At: sqliteBase().Add(time.Duration(i) * time.Second)}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := st.DeleteRecentMessages(ctx, "sess", 1); err != nil {
|
||||
t.Fatalf("DeleteRecentMessages: %v", err)
|
||||
}
|
||||
|
||||
if err := ftsIntegrityErr(t, st); err != nil {
|
||||
t.Fatalf("FTS5 integrity check failed after trigger-maintained delete: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestSQLiteSessionStore_HostileQueriesAreLiteral pins that free-text input
|
||||
// is never executed as FTS5 query syntax: column filters, the NEAR operator,
|
||||
// prefix wildcards, and boolean operators must all be treated as literal
|
||||
// terms (or, for a bare "*", an empty phrase) without syntax errors. Each
|
||||
// row asserts the exact match set, not merely the absence of a panic.
|
||||
func TestSQLiteSessionStore_HostileQueriesAreLiteral(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := newTestSQLiteStore(t)
|
||||
|
||||
seed := []string{
|
||||
"text:hello here", // phrase (text, hello)
|
||||
"hello there", // would match a raw "text:hello" column filter
|
||||
"hello world",
|
||||
`hello "world quote`, // unmatched-quote input must still find this
|
||||
"near a b", // phrase (near, a) AND token b
|
||||
"a far b", // would match a raw NEAR(a b) operator
|
||||
"star * symbol", // "*" is a separator: never a token, never matches
|
||||
"needle or elsewhere",
|
||||
"needle elsewhere", // would match a raw OR query
|
||||
"needle",
|
||||
"alpha beta gamma", // both AND terms present
|
||||
"alpha alone",
|
||||
"beta alone",
|
||||
}
|
||||
for i, text := range seed {
|
||||
if err := st.SaveMessage(ctx, "sess", Message{From: "u", Text: text, At: sqliteBase().Add(time.Duration(i) * time.Second)}); err != nil {
|
||||
t.Fatalf("SaveMessage %q: %v", text, err)
|
||||
}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
want []string
|
||||
}{
|
||||
{"column syntax is a literal phrase, not a column filter", "text:hello", []string{"text:hello here"}},
|
||||
{"unmatched quote is literal, not a syntax error", `hello "world`, []string{"hello world", `hello "world quote`}},
|
||||
{"NEAR syntax is literal, not the NEAR operator", "NEAR(a b)", []string{"near a b"}},
|
||||
{"bare wildcard is an empty phrase, not a prefix operator", "*", nil},
|
||||
{"OR is a literal term, not the boolean operator", "needle OR elsewhere", []string{"needle or elsewhere"}},
|
||||
{"multi-term query pins AND semantics", "alpha beta", []string{"alpha beta gamma"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
page, err := st.SearchMessages(ctx, "sess", MessageQuery{Query: tc.query, Limit: 100})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages(%q): %v", tc.query, err)
|
||||
}
|
||||
if !sameTexts(texts(page.Messages), tc.want) {
|
||||
t.Errorf("query %q matched %v, want %v", tc.query, texts(page.Messages), tc.want)
|
||||
}
|
||||
if page.Truncated || page.HasMore {
|
||||
t.Errorf("query %q: Truncated=%v HasMore=%v, want false/false", tc.query, page.Truncated, page.HasMore)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFTSMatchQueryEscapesHostileInput pins the exact MATCH expression the
|
||||
// store generates: every whitespace-separated term becomes a quoted phrase
|
||||
// (embedded quotes doubled), joined by explicit AND, so user input can never
|
||||
// reach the FTS5 parser as query syntax.
|
||||
func TestFTSMatchQueryEscapesHostileInput(t *testing.T) {
|
||||
tests := []struct{ in, want string }{
|
||||
{"", ""},
|
||||
{" ", ""},
|
||||
{"alpha beta", `"alpha" AND "beta"`},
|
||||
{"needle OR elsewhere", `"needle" AND "OR" AND "elsewhere"`},
|
||||
{"text:hello", `"text:hello"`},
|
||||
{`hello "world`, `"hello" AND """world"`},
|
||||
{"NEAR(a b)", `"NEAR(a" AND "b)"`},
|
||||
{"*", `"*"`},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if got := ftsMatchQuery(tc.in); got != tc.want {
|
||||
t.Errorf("ftsMatchQuery(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package gateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/samcharles93/NellDB/logstore"
|
||||
)
|
||||
|
||||
func TestSessionStore_SaveAndGet(t *testing.T) {
|
||||
@@ -339,3 +342,57 @@ func TestSessionStore_List(t *testing.T) {
|
||||
t.Fatalf("expected 3 sessions, got %d", len(all))
|
||||
}
|
||||
}
|
||||
|
||||
// TestNellSessionStore_SourceIDSurvivesLogReopen proves the persisted
|
||||
// source_id representation survives the log store's JSON round-trip: after
|
||||
// a close and reopen of the same file, the freshly rebuilt DocDB decodes
|
||||
// the byte-value array back into the original UTF-8 string, and the
|
||||
// correlation ID is still invisible to SearchMessages. Reopen is the path
|
||||
// that exercises the JSON-decoded []any-of-float64 shape rather than the
|
||||
// in-process []int shape.
|
||||
func TestNellSessionStore_SourceIDSurvivesLogReopen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
logPath := filepath.Join(t.TempDir(), "messages.log")
|
||||
|
||||
const sourceID = "tg-идентификатор-42"
|
||||
open := func() SessionStore {
|
||||
ls, err := logstore.OpenLog(logPath, "test-node")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenLog: %v", err)
|
||||
}
|
||||
return NewSessionStore(ls, "test-node")
|
||||
}
|
||||
|
||||
st := open()
|
||||
if err := st.SaveMessage(ctx, "sess", Message{SourceID: sourceID, From: "u", Text: "hello world", At: base}); err != nil {
|
||||
t.Fatalf("SaveMessage: %v", err)
|
||||
}
|
||||
if err := st.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
st2 := open()
|
||||
defer func() { _ = st2.Close() }()
|
||||
|
||||
got, err := st2.RecentMessages(ctx, "sess", 10)
|
||||
if err != nil {
|
||||
t.Fatalf("RecentMessages after reopen: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("got %d messages after reopen, want 1", len(got))
|
||||
}
|
||||
if got[0].SourceID != sourceID {
|
||||
t.Errorf("SourceID after reopen = %q, want %q", got[0].SourceID, sourceID)
|
||||
}
|
||||
if got[0].MessageID == "" || got[0].MessageID == sourceID {
|
||||
t.Errorf("canonical MessageID = %q after reopen, want non-empty and distinct from SourceID", got[0].MessageID)
|
||||
}
|
||||
|
||||
page, err := st2.SearchMessages(ctx, "sess", MessageQuery{Query: sourceID, Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("SearchMessages after reopen: %v", err)
|
||||
}
|
||||
if len(page.Messages) != 0 {
|
||||
t.Errorf("search for SourceID %q after reopen matched %v, want no matches", sourceID, texts(page.Messages))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -131,7 +131,8 @@ func (a *Adapter) EnqueueIssue(ctx context.Context, owner, repo string, number i
|
||||
"park_reason": "",
|
||||
"retry_count": int64(0),
|
||||
"watch_comment_id": int64(0),
|
||||
"created_at": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"created_at": nowRFC3339(),
|
||||
"updated_at": nowRFC3339(),
|
||||
}
|
||||
if _, err := a.tasks.Put(ctx, doc); err != nil {
|
||||
return false, fmt.Errorf("nell: enqueue: %w", err)
|
||||
@@ -178,7 +179,8 @@ func (a *Adapter) EnqueueChatTask(ctx context.Context, owner, repo, title, body,
|
||||
"watch_comment_id": int64(0),
|
||||
"source": store.SourceChat,
|
||||
"identity": identity,
|
||||
"created_at": time.Now().UTC().Format(time.RFC3339Nano),
|
||||
"created_at": nowRFC3339(),
|
||||
"updated_at": nowRFC3339(),
|
||||
}
|
||||
if _, err := a.tasks.Put(ctx, doc); err != nil {
|
||||
return nil, fmt.Errorf("nell: enqueue chat task: %w", err)
|
||||
@@ -305,6 +307,7 @@ func (a *Adapter) Update(ctx context.Context, t *store.Task) error {
|
||||
doc["iterations"] = int64(t.Iterations)
|
||||
doc["park_reason"] = t.ParkReason
|
||||
doc["watch_comment_id"] = t.WatchCommentID
|
||||
doc["updated_at"] = nowRFC3339()
|
||||
if _, err := a.tasks.Put(ctx, doc); err != nil {
|
||||
return fmt.Errorf("nell: update task: %w", err)
|
||||
}
|
||||
@@ -796,6 +799,8 @@ func docToTask(doc sdk.Doc) *store.Task {
|
||||
Body: strField(doc, "body"),
|
||||
Labels: strField(doc, "labels"),
|
||||
Status: strField(doc, "status"),
|
||||
CreatedAt: timeField(doc, "created_at"),
|
||||
UpdatedAt: timeField(doc, "updated_at"),
|
||||
Workflow: strField(doc, "workflow"),
|
||||
Stage: strField(doc, "stage"),
|
||||
Branch: strField(doc, "branch"),
|
||||
@@ -886,6 +891,7 @@ func (a *Adapter) putTask(ctx context.Context, key string, t *store.Task) error
|
||||
}
|
||||
}
|
||||
taskFields(t, doc)
|
||||
doc["updated_at"] = nowRFC3339()
|
||||
if _, err := a.tasks.Put(ctx, doc); err != nil {
|
||||
return fmt.Errorf("nell: put task: %w", err)
|
||||
}
|
||||
@@ -993,3 +999,25 @@ func clip(s string, n int) string {
|
||||
}
|
||||
return s[:pos]
|
||||
}
|
||||
|
||||
// nowRFC3339 is the timestamp format task documents use. Kept in one place so
|
||||
// created_at and updated_at cannot drift apart in format.
|
||||
func nowRFC3339() string {
|
||||
return time.Now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// timeField parses an RFC3339 document field, returning the zero time when it
|
||||
// is absent or unparseable. Documents written before these fields existed
|
||||
// simply report a zero timestamp rather than failing the read.
|
||||
func timeField(doc sdk.Doc, key string) time.Time {
|
||||
v := strField(doc, key)
|
||||
if v == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339} {
|
||||
if parsed, err := time.Parse(layout, v); err == nil {
|
||||
return parsed.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
Reference in new issue
Block a user