Remove gateway re-exports of messaging helpers with no callers, the unused DeltaFunc adapter, DefaultPersonas, NewStoreTaskCreator, the SortBySeq stub, and the in-memory session store that nothing constructs.
263 lines
8.2 KiB
Go
263 lines
8.2 KiB
Go
package gateway
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// CompressionConfig controls automatic context compression for long-running
|
|
// chat sessions. Compression keeps the conversation history within the model's
|
|
// context window by summarising older messages when the total estimated token
|
|
// count exceeds a threshold.
|
|
type CompressionConfig struct {
|
|
// Enabled toggles automatic compression. When false, compression is
|
|
// skipped regardless of other settings.
|
|
Enabled bool
|
|
|
|
// Threshold is the fraction of the context window at which compression
|
|
// triggers. For example, 0.5 means compress when estimated tokens
|
|
// exceed 50% of ContextWindow. Must be between 0 and 1.
|
|
Threshold float64
|
|
|
|
// ContextWindow is the model's maximum context size in tokens (not
|
|
// characters). Used with Threshold to decide when to compress.
|
|
ContextWindow int
|
|
|
|
// MaxPromptTokens is the effective history budget after reserving space
|
|
// for the system prompt, tools, and model output. A positive value takes
|
|
// precedence over ContextWindow*Threshold.
|
|
MaxPromptTokens int
|
|
|
|
// ProtectFirst keeps the first N messages unmodified (typically the
|
|
// system greeting and initial instructions).
|
|
ProtectFirst int
|
|
|
|
// ProtectLast keeps the last N messages unmodified (the most recent
|
|
// conversation turns).
|
|
ProtectLast int
|
|
|
|
// SummaryMarker is inserted between the truncated middle and the
|
|
// protected tail to indicate compression occurred.
|
|
SummaryMarker string
|
|
}
|
|
|
|
// DefaultCompressionConfig returns a CompressionConfig with sensible
|
|
// defaults: compression at 50% threshold, protecting
|
|
// first 3 and last 20 messages. ContextWindow defaults to 128k.
|
|
func DefaultCompressionConfig() CompressionConfig {
|
|
return CompressionConfig{
|
|
Enabled: true,
|
|
Threshold: 0.5,
|
|
ContextWindow: 128000,
|
|
ProtectFirst: 3,
|
|
ProtectLast: 20,
|
|
SummaryMarker: "[Earlier conversation has been summarised to stay within context limits.]",
|
|
}
|
|
}
|
|
|
|
// CompressionConfigForModel derives the history budget from the selected
|
|
// model rather than assuming every provider has the default context window.
|
|
// promptReserve is the estimated system/tool prompt cost already known before
|
|
// history is assembled.
|
|
func CompressionConfigForModel(details ModelDetails, promptReserve int) (CompressionConfig, error) {
|
|
cfg := DefaultCompressionConfig()
|
|
if details.ContextWindow <= 0 {
|
|
// ModelDetails is an optional extension on ModelManager. Preserve
|
|
// compatibility for minimal adapters, while production catalog-backed
|
|
// managers still take the model-specific path below.
|
|
cfg.MaxPromptTokens = max(cfg.ContextWindow-max(promptReserve, 0), 1)
|
|
return cfg, nil
|
|
}
|
|
|
|
reserve := max(promptReserve, 0) + max(details.MaxOutputTokens, 0)
|
|
if reserve >= details.ContextWindow {
|
|
return CompressionConfig{}, fmt.Errorf("model %q context window %d is smaller than reserved prompt/output budget %d", details.Ref, details.ContextWindow, reserve)
|
|
}
|
|
cfg.ContextWindow = details.ContextWindow
|
|
cfg.MaxPromptTokens = max(details.ContextWindow-reserve, 1)
|
|
return cfg, nil
|
|
}
|
|
|
|
// tokenEstimate estimates a string's tokens: four ASCII characters per token,
|
|
// one token per non-ASCII rune.
|
|
func tokenEstimate(s string) int {
|
|
ascii, nonASCII := 0, 0
|
|
for _, r := range s {
|
|
if r < 128 {
|
|
ascii++
|
|
continue
|
|
}
|
|
nonASCII++
|
|
}
|
|
return ascii/4 + nonASCII
|
|
}
|
|
|
|
// EstimateTokens exposes the same conservative estimate used by compression
|
|
// so callers can reserve space for system prompts and tool descriptions.
|
|
func EstimateTokens(s string) int {
|
|
return tokenEstimate(s)
|
|
}
|
|
|
|
// CompressedView holds the compressed representation of a conversation
|
|
// history.
|
|
type CompressedView struct {
|
|
// Messages is the compressed message list ready for the LLM.
|
|
Messages []CompressedMessage
|
|
|
|
// WasCompressed is true when compression was applied.
|
|
WasCompressed bool
|
|
|
|
// TokensBefore is the estimated token count before compression.
|
|
TokensBefore int
|
|
|
|
// TokensAfter is the estimated token count after compression.
|
|
TokensAfter int
|
|
|
|
// ProtectedFirst and ProtectedLast index the input messages kept unchanged;
|
|
// meaningful only when WasCompressed is true.
|
|
ProtectedFirst int
|
|
ProtectedLast int
|
|
}
|
|
|
|
// CompressedMessage is a message in a compressed conversation view.
|
|
type CompressedMessage struct {
|
|
Role string
|
|
Content string
|
|
}
|
|
|
|
// CompressHistory replaces the messages between ProtectFirst and ProtectLast
|
|
// with a summary marker once the estimated token count reaches the threshold.
|
|
func CompressHistory(messages []CompressedMessage, cfg CompressionConfig) CompressedView {
|
|
if !cfg.Enabled || cfg.ContextWindow <= 0 {
|
|
return CompressedView{
|
|
Messages: messages,
|
|
TokensAfter: estimateTotal(messages),
|
|
}
|
|
}
|
|
|
|
totalTokens := estimateTotal(messages)
|
|
threshold := cfg.MaxPromptTokens
|
|
if threshold <= 0 {
|
|
threshold = int(float64(cfg.ContextWindow) * cfg.Threshold)
|
|
}
|
|
|
|
if totalTokens <= threshold || len(messages) == 0 {
|
|
return CompressedView{
|
|
Messages: messages,
|
|
TokensBefore: totalTokens,
|
|
TokensAfter: totalTokens,
|
|
}
|
|
}
|
|
|
|
// Compression: keep first N + summary marker + last N. If the protected
|
|
// tail itself exceeds the effective budget, reduce it until the resulting
|
|
// view fits. This matters for small-context local models.
|
|
protectLast := min(cfg.ProtectLast, len(messages))
|
|
protectFirst := min(cfg.ProtectFirst, len(messages)-protectLast)
|
|
for {
|
|
truncated := messages[protectFirst : len(messages)-protectLast]
|
|
summary := summariseTruncated(truncated, cfg)
|
|
|
|
compressed := make([]CompressedMessage, 0, protectFirst+1+protectLast)
|
|
compressed = append(compressed, messages[:protectFirst]...)
|
|
compressed = append(compressed, CompressedMessage{Role: "system", Content: summary})
|
|
compressed = append(compressed, messages[len(messages)-protectLast:]...)
|
|
compressedTokens := estimateTotal(compressed)
|
|
if compressedTokens <= threshold || (protectFirst == 0 && protectLast == 0) {
|
|
if compressedTokens > threshold && protectFirst == 0 && protectLast == 0 {
|
|
maxSummaryChars := max(threshold*4, 1)
|
|
compressed[0].Content = truncate(compressed[0].Content, maxSummaryChars)
|
|
compressedTokens = estimateTotal(compressed)
|
|
}
|
|
return CompressedView{
|
|
Messages: compressed,
|
|
WasCompressed: true,
|
|
TokensBefore: totalTokens,
|
|
TokensAfter: compressedTokens,
|
|
ProtectedFirst: protectFirst,
|
|
ProtectedLast: protectLast,
|
|
}
|
|
}
|
|
if protectLast > 0 {
|
|
protectLast--
|
|
continue
|
|
}
|
|
protectFirst--
|
|
}
|
|
}
|
|
|
|
// SummaryIndex returns the position of the generated summary within Messages,
|
|
// or -1 when nothing was compressed. The summary is the one element of a
|
|
// compressed view that is genuinely new; everything else corresponds to an
|
|
// input message at a known offset.
|
|
func (v CompressedView) SummaryIndex() int {
|
|
if !v.WasCompressed {
|
|
return -1
|
|
}
|
|
return v.ProtectedFirst
|
|
}
|
|
|
|
// estimateTotal returns the sum of estimated token counts across messages.
|
|
func estimateTotal(messages []CompressedMessage) int {
|
|
total := 0
|
|
for _, m := range messages {
|
|
total += tokenEstimate(m.Content)
|
|
}
|
|
return total
|
|
}
|
|
|
|
// summariseTruncated produces a summary of messages removed during
|
|
// compression. Without an LLM summarisation call (deferred to the
|
|
// memory provider), we produce a statistical summary that tells the
|
|
// model what was removed.
|
|
func summariseTruncated(messages []CompressedMessage, cfg CompressionConfig) string {
|
|
if len(messages) == 0 {
|
|
return cfg.SummaryMarker
|
|
}
|
|
|
|
// Count turns by role.
|
|
var userTurns, assistantTurns int
|
|
for _, m := range messages {
|
|
switch {
|
|
case strings.EqualFold(m.Role, "user"):
|
|
userTurns++
|
|
case strings.EqualFold(m.Role, "assistant"):
|
|
assistantTurns++
|
|
}
|
|
}
|
|
|
|
// Find the topics from the first and last message.
|
|
first := truncate(messages[0].Content, 80)
|
|
last := truncate(messages[len(messages)-1].Content, 80)
|
|
|
|
// Estimate tokens removed.
|
|
removedTokens := estimateTotal(messages)
|
|
|
|
return fmt.Sprintf(
|
|
"%s\n\n[%d messages removed (~%d tokens): %d user turns, %d assistant turns. "+
|
|
"From: %q ... To: %q]",
|
|
cfg.SummaryMarker,
|
|
len(messages),
|
|
removedTokens,
|
|
userTurns,
|
|
assistantTurns,
|
|
first,
|
|
last,
|
|
)
|
|
}
|
|
|
|
// truncate cuts s to at most n runes.
|
|
func truncate(s string, n int) string {
|
|
if n <= 0 {
|
|
return ""
|
|
}
|
|
runes := []rune(s)
|
|
if len(runes) <= n {
|
|
return s
|
|
}
|
|
if n <= 3 {
|
|
return string(runes[:n])
|
|
}
|
|
return string(runes[:n-3]) + "..."
|
|
}
|