Files
archie/internal/infrastructure/memory/builtin_engine.go
T

477 lines
15 KiB
Go

// Package memory implements domain/memory.MemoryEngine over concrete backends.
// The builtin engine persists each scope's records as markdown under its root
// -- human-readable and hand-editable -- with a per-scope HISTORY.md retaining
// the states that are no longer live.
package memory
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/google/uuid"
domainmemory "github.com/samcharles93/archie-core/internal/domain/memory"
"github.com/samcharles93/archie-core/internal/infrastructure/memory/builtin"
)
// EngineName is the name BuiltinEngine registers under and the default the
// config surface resolves to when no engine is configured.
const EngineName = "builtin"
// defaultSectionName is the section a record with an empty Kind is written
// to.
const defaultSectionName = "general"
// liveFileName is the one live markdown document per scope, holding that
// scope's heads.
const liveFileName = "OBSERVATIONS.md"
// defaultQueryLimit bounds Query when the caller names no limit: 0 in
// Query.Limit means "the engine's default", never unlimited, and List is the
// unbounded read for one scope.
const defaultQueryLimit = 100
// contentScanner is the family's content scanner, applied in Create and
// Update. Those two are the one choke point every producer crosses, so the
// curator's model-extracted content is covered as well as the chat tool's
var contentScanner domainmemory.Scanner = &domainmemory.DefaultScanner{}
// scopeStores is one scope's on-disk pair: the heads, and the states that
// used to be heads.
type scopeStores struct {
live *builtin.Store
history *builtin.Store
}
// BuiltinEngine implements domain/memory.MemoryEngine over one pair of
// builtin.Store documents per scope.
type BuiltinEngine struct {
root string
maxFileBytes int
mu sync.Mutex
stores map[string]*scopeStores // keyed by scope key, lazily created
clock domainmemory.Clock
log *slog.Logger
}
// NewBuiltinEngine stores each scope under root/hex(sha256(scope.Key())).
// maxFileBytes <= 0 uses the store default.
func NewBuiltinEngine(root string, maxFileBytes int) *BuiltinEngine {
return &BuiltinEngine{root: root, maxFileBytes: maxFileBytes, stores: make(map[string]*scopeStores)}
}
func (e *BuiltinEngine) Name() string { return EngineName }
func (e *BuiltinEngine) Version() string { return "1" }
func (e *BuiltinEngine) Manifest() domainmemory.Manifest {
return domainmemory.Manifest{RequiresNetwork: false}
}
// Bind takes the registrar's clock and logger.
func (e *BuiltinEngine) Bind(host domainmemory.Registrar) {
e.clock = host.Clock
e.log = host.Log
}
func (e *BuiltinEngine) Start(context.Context) error { return nil }
func (e *BuiltinEngine) Health(context.Context) domainmemory.Health {
return domainmemory.Health{Status: domainmemory.HealthHealthy}
}
func (e *BuiltinEngine) Stop(context.Context) error { return nil }
// systemClock is the fallback for an engine no registry bound. Registrar's
// own doc says a nil Clock is replaced at construction, but a directly
// constructed engine (a test, an embedder) has no registry at all, and a
// clock nil-panic on a write would be a poor way to find that out.
type systemClock struct{}
func (systemClock) Now() time.Time { return time.Now() }
var fallbackClock domainmemory.Clock = systemClock{}
func (e *BuiltinEngine) now() time.Time {
if e.clock == nil {
return fallbackClock.Now()
}
return e.clock.Now()
}
// ── storage ────────────────────────────────────────────────────────────
// scopeStoresFor returns the scope's cached document pair. create is false
// for reads, which never create a directory. scope must be valid.
func (e *BuiltinEngine) scopeStoresFor(scope domainmemory.Scope, create bool) (*scopeStores, bool, error) {
key := scope.Key()
e.mu.Lock()
defer e.mu.Unlock()
if s, ok := e.stores[key]; ok {
return s, true, nil
}
dir := filepath.Join(e.root, scopeDirName(scope))
if create {
if err := os.MkdirAll(dir, 0o755); err != nil {
return nil, false, fmt.Errorf("memory: builtin engine: scope directory %s unusable: %w", dir, err)
}
} else if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
return nil, false, nil
}
return nil, false, fmt.Errorf("memory: builtin engine: scope directory %s unusable: %w", dir, err)
}
live, err := builtin.NewStore(filepath.Join(dir, liveFileName), e.maxFileBytes)
if err != nil {
return nil, false, err
}
history, err := builtin.NewStore(filepath.Join(dir, historyFileName), historyMaxFileBytes)
if err != nil {
return nil, false, err
}
stores := &scopeStores{live: live, history: history}
e.stores[key] = stores
return stores, true, nil
}
// scopeDirName maps a scope to a filesystem-safe, collision-resistant
// directory name.
func scopeDirName(scope domainmemory.Scope) string {
sum := sha256.Sum256([]byte(scope.Key()))
return hex.EncodeToString(sum[:])
}
// sectionFor returns the section a record's Kind maps to, validated the same
// way builtin validates any section name -- so an invalid Kind fails with the
// error shape the memory tool already returns for an invalid section, not a
// new one.
func sectionFor(kind string) (string, error) {
if kind == "" {
kind = defaultSectionName
}
if err := builtin.ValidateSectionName(kind); err != nil {
return "", err
}
return kind, nil
}
// scanContent rejects content with a block-level scanner hit and logs
// warn-level hits.
func (e *BuiltinEngine) scanContent(content string) error {
result := contentScanner.ScanContent(content)
switch result.Level {
case domainmemory.ThreatBlock:
return fmt.Errorf("memory: builtin engine: refusing to persist content: %s", result.Message)
case domainmemory.ThreatWarn:
if e.log != nil {
e.log.Warn("memory: stored content matched a sensitive-data pattern",
"pattern", result.Pattern, "reason", result.Message)
}
}
return nil
}
// storeContent returns content as it will be stored: trimmed, and rejected
// when empty.
func storeContent(content string) (string, error) {
trimmed := strings.TrimSpace(content)
if trimmed == "" {
return "", errors.New("memory: builtin engine: content must not be empty")
}
return trimmed, nil
}
// findBlock returns the live block addressed by id.
func findBlock(store *builtin.Store, id domainmemory.RecordID) (parsedBlock, bool) {
for _, block := range parseBlocks(store.Render()) {
if block.marker.ID == string(id) {
return block, true
}
}
return parsedBlock{}, false
}
func notFound(scope domainmemory.Scope, id domainmemory.RecordID) error {
return fmt.Errorf("memory: builtin engine: %w: id %q in scope %s", domainmemory.ErrNotFound, id, scope)
}
// ── Store ──────────────────────────────────────────────────────────────
// Create records one new memory: the id is assigned once, here, and is the
// only address the record ever has.
func (e *BuiltinEngine) Create(_ context.Context, in domainmemory.NewRecord) (domainmemory.Record, error) {
if err := in.Validate(); err != nil {
return domainmemory.Record{}, err
}
section, err := sectionFor(in.Kind)
if err != nil {
return domainmemory.Record{}, err
}
if err := e.scanContent(in.Content); err != nil {
return domainmemory.Record{}, err
}
content, err := storeContent(in.Content)
if err != nil {
return domainmemory.Record{}, err
}
stores, _, err := e.scopeStoresFor(in.Scope, true)
if err != nil {
return domainmemory.Record{}, err
}
now := e.now()
marker := markerData{
ID: uuid.NewString(),
Revision: 1,
Kind: section,
Author: in.Author,
OriginUser: string(in.OriginUser),
Source: in.Source,
CreatedAt: now,
UpdatedAt: now,
}
block, err := renderBlock(marker, content)
if err != nil {
return domainmemory.Record{}, err
}
for {
_, err := stores.live.Add(section, block)
if !errors.Is(err, builtin.ErrTooLarge) {
if err != nil {
return domainmemory.Record{}, err
}
return recordOf(marker, in.Scope, content), nil
}
if err := e.retireOldest(stores, section); err != nil {
return domainmemory.Record{}, err
}
}
}
// retireOldest moves the oldest record of a full section to history, so a
// section at the size cap keeps recording instead of refusing every write.
func (e *BuiltinEngine) retireOldest(stores *scopeStores, section string) error {
for _, block := range parseBlocks(stores.live.Render()) {
if block.section != section || block.marker.ID == "" {
continue
}
if err := appendHistory(stores.history, block.marker, block.content(), e.now(), true); err != nil {
return err
}
_, err := stores.live.Remove(section, block.text, true)
return err
}
return fmt.Errorf("memory: builtin engine: record larger than section %q can hold: %w", section, builtin.ErrTooLarge)
}
func (e *BuiltinEngine) Get(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) (domainmemory.Record, error) {
if err := scope.Validate(); err != nil {
return domainmemory.Record{}, err
}
if id == "" {
return domainmemory.Record{}, errors.New("memory: builtin engine: id must not be empty")
}
stores, ok, err := e.scopeStoresFor(scope, false)
if err != nil {
return domainmemory.Record{}, err
}
if !ok {
return domainmemory.Record{}, notFound(scope, id)
}
block, ok := findBlock(stores.live, id)
if !ok {
return domainmemory.Record{}, notFound(scope, id)
}
return block.record(scope), nil
}
// List returns one scope's heads, most recently updated first.
func (e *BuiltinEngine) List(_ context.Context, scope domainmemory.Scope) ([]domainmemory.Record, error) {
if err := scope.Validate(); err != nil {
return nil, err
}
stores, ok, err := e.scopeStoresFor(scope, false)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
blocks := parseBlocks(stores.live.Render())
records := make([]domainmemory.Record, 0, len(blocks))
for _, block := range blocks {
records = append(records, block.record(scope))
}
return orderHeads(records), nil
}
// orderHeads returns heads most recently updated first, ties in creation
// order.
func orderHeads(records []domainmemory.Record) []domainmemory.Record {
slices.Reverse(records)
slices.SortStableFunc(records, func(a, b domainmemory.Record) int {
return b.UpdatedAt.Compare(a.UpdatedAt)
})
return records
}
// Query returns the heads of exactly the scopes q names -- the caller's read
// set, never widened here -- grouped in the order those scopes were given and
// bounded by q.Limit across the whole result.
func (e *BuiltinEngine) Query(ctx context.Context, q domainmemory.Query) ([]domainmemory.Record, error) {
if err := q.Validate(); err != nil {
return nil, err
}
limit := q.Limit
if limit == 0 {
limit = defaultQueryLimit
}
var out []domainmemory.Record
for _, scope := range q.Scopes {
heads, err := e.List(ctx, scope)
if err != nil {
return nil, err
}
for _, record := range heads {
if q.Text != "" && !strings.Contains(record.Content, q.Text) {
continue
}
out = append(out, record)
if len(out) == limit {
return out, nil
}
}
}
return out, nil
}
// Update supersedes a record's content in place and returns the new state.
//
// A record may not move between scopes, which is why RecordUpdate carries no
// scope to change: it is addressed by the scope it is already in.
func (e *BuiltinEngine) Update(_ context.Context, in domainmemory.RecordUpdate) (domainmemory.Record, error) {
if err := in.Validate(); err != nil {
return domainmemory.Record{}, err
}
// Scanned before anything is written: a refused update must leave both
// the record and its history untouched.
if err := e.scanContent(in.Content); err != nil {
return domainmemory.Record{}, err
}
content, err := storeContent(in.Content)
if err != nil {
return domainmemory.Record{}, err
}
stores, ok, err := e.scopeStoresFor(in.Scope, false)
if err != nil {
return domainmemory.Record{}, err
}
if !ok {
return domainmemory.Record{}, notFound(in.Scope, in.ID)
}
// Reload from disk before checking Expected.
if err := stores.live.Reload(); err != nil {
return domainmemory.Record{}, err
}
current, ok := findBlock(stores.live, in.ID)
if !ok {
return domainmemory.Record{}, notFound(in.Scope, in.ID)
}
// Expected is what makes a lost update detectable: the revision the
// caller last saw is the check, against the revision just read.
if in.Expected != 0 && in.Expected != current.marker.Revision {
return domainmemory.Record{}, fmt.Errorf(
"memory: builtin engine: %w: record %q is at revision %d, not %d",
domainmemory.ErrStaleRevision, in.ID, current.marker.Revision, in.Expected,
)
}
now := e.now()
next := current.marker
next.Revision = current.marker.Revision + 1
next.Author = in.Author
next.Source = in.Source
next.UpdatedAt = now
// Crash safety, in this order: the superseded state is durable in
// HISTORY.md before the live block is touched. A crash between the two
// leaves an extra history entry and an unchanged record -- recoverable,
if err := appendHistory(stores.history, current.marker, current.content(), now, false); err != nil {
return domainmemory.Record{}, err
}
block, err := renderBlock(next, content)
if err != nil {
return domainmemory.Record{}, err
}
if _, err := stores.live.Replace(current.section, current.text, block, true); err != nil {
return domainmemory.Record{}, err
}
return recordOf(next, in.Scope, content), nil
}
// Forget removes a record's live state, retaining a deleted revision so its
// provenance outlives its content. Forgetting an id that is not there is not
// an error: the end state, id absent, already holds.
func (e *BuiltinEngine) Forget(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) error {
if err := scope.Validate(); err != nil {
return err
}
if id == "" {
return errors.New("memory: builtin engine: id must not be empty")
}
stores, ok, err := e.scopeStoresFor(scope, false)
if err != nil {
return err
}
if !ok {
return nil
}
current, ok := findBlock(stores.live, id)
if !ok {
return nil
}
// The same crash-safe order as Update: the deleted revision is durable
// before the live block disappears, so the record's provenance is never
// the part that is lost.
if err := appendHistory(stores.history, current.marker, current.content(), e.now(), true); err != nil {
return err
}
_, err = stores.live.Remove(current.section, current.text, true)
return err
}
// Revisions returns a record's retained states, oldest first, including the
// deleted revision Forget records. A record that never existed is empty, not
// an error.
func (e *BuiltinEngine) Revisions(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) ([]domainmemory.Revision, error) {
if err := scope.Validate(); err != nil {
return nil, err
}
if id == "" {
return nil, errors.New("memory: builtin engine: id must not be empty")
}
stores, ok, err := e.scopeStoresFor(scope, false)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
return revisionsOf(stores.history, scope, id), nil
}