453 lines
14 KiB
Go
453 lines
14 KiB
Go
// Package memory implements domain/memory.MemoryEngine over concrete backends.
|
|
// The builtin engine persists each scope's records as markdown under its root
|
|
// -- human-readable and hand-editable -- with a per-scope HISTORY.md retaining
|
|
// the states that are no longer live.
|
|
package memory
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
domainmemory "github.com/samcharles93/archie-core/internal/domain/memory"
|
|
"github.com/samcharles93/archie-core/internal/infrastructure/memory/builtin"
|
|
)
|
|
|
|
// EngineName is the name BuiltinEngine registers under and the default the
|
|
// config surface resolves to when no engine is configured.
|
|
const EngineName = "builtin"
|
|
|
|
// defaultSectionName is the section a record with an empty Kind is written
|
|
// to.
|
|
const defaultSectionName = "general"
|
|
|
|
// liveFileName is the one live markdown document per scope, holding that
|
|
// scope's heads.
|
|
const liveFileName = "OBSERVATIONS.md"
|
|
|
|
// defaultQueryLimit bounds Query when the caller names no limit: 0 in
|
|
// Query.Limit means "the engine's default", never unlimited, and List is the
|
|
// unbounded read for one scope.
|
|
const defaultQueryLimit = 100
|
|
|
|
// contentScanner is the family's content scanner, applied in Create and
|
|
// Update. Those two are the one choke point every producer crosses, so the
|
|
// curator's model-extracted content is covered as well as the chat tool's
|
|
var contentScanner domainmemory.Scanner = &domainmemory.DefaultScanner{}
|
|
|
|
// scopeStores is one scope's on-disk pair: the heads, and the states that
|
|
// used to be heads.
|
|
type scopeStores struct {
|
|
live *builtin.Store
|
|
history *builtin.Store
|
|
}
|
|
|
|
// BuiltinEngine implements domain/memory.MemoryEngine over one pair of
|
|
// builtin.Store documents per scope.
|
|
type BuiltinEngine struct {
|
|
root string
|
|
maxFileBytes int
|
|
|
|
mu sync.Mutex
|
|
stores map[string]*scopeStores // keyed by scope key, lazily created
|
|
|
|
clock domainmemory.Clock
|
|
log *slog.Logger
|
|
}
|
|
|
|
// NewBuiltinEngine stores each scope under root/hex(sha256(scope.Key())).
|
|
// maxFileBytes <= 0 uses the store default.
|
|
func NewBuiltinEngine(root string, maxFileBytes int) *BuiltinEngine {
|
|
return &BuiltinEngine{root: root, maxFileBytes: maxFileBytes, stores: make(map[string]*scopeStores)}
|
|
}
|
|
|
|
func (e *BuiltinEngine) Name() string { return EngineName }
|
|
func (e *BuiltinEngine) Version() string { return "1" }
|
|
|
|
func (e *BuiltinEngine) Manifest() domainmemory.Manifest {
|
|
return domainmemory.Manifest{RequiresNetwork: false}
|
|
}
|
|
|
|
// Bind takes the registrar's clock and logger.
|
|
func (e *BuiltinEngine) Bind(host domainmemory.Registrar) {
|
|
e.clock = host.Clock
|
|
e.log = host.Log
|
|
}
|
|
|
|
func (e *BuiltinEngine) Start(context.Context) error { return nil }
|
|
|
|
func (e *BuiltinEngine) Health(context.Context) domainmemory.Health {
|
|
return domainmemory.Health{Status: domainmemory.HealthHealthy}
|
|
}
|
|
|
|
func (e *BuiltinEngine) Stop(context.Context) error { return nil }
|
|
|
|
// systemClock is the fallback for an engine no registry bound. Registrar's
|
|
// own doc says a nil Clock is replaced at construction, but a directly
|
|
// constructed engine (a test, an embedder) has no registry at all, and a
|
|
// clock nil-panic on a write would be a poor way to find that out.
|
|
type systemClock struct{}
|
|
|
|
func (systemClock) Now() time.Time { return time.Now() }
|
|
|
|
var fallbackClock domainmemory.Clock = systemClock{}
|
|
|
|
func (e *BuiltinEngine) now() time.Time {
|
|
if e.clock == nil {
|
|
return fallbackClock.Now()
|
|
}
|
|
return e.clock.Now()
|
|
}
|
|
|
|
// ── storage ────────────────────────────────────────────────────────────
|
|
|
|
// scopeStoresFor returns the scope's cached document pair. create is false
|
|
// for reads, which never create a directory. scope must be valid.
|
|
func (e *BuiltinEngine) scopeStoresFor(scope domainmemory.Scope, create bool) (*scopeStores, bool, error) {
|
|
key := scope.Key()
|
|
e.mu.Lock()
|
|
defer e.mu.Unlock()
|
|
if s, ok := e.stores[key]; ok {
|
|
return s, true, nil
|
|
}
|
|
|
|
dir := filepath.Join(e.root, scopeDirName(scope))
|
|
if create {
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return nil, false, fmt.Errorf("memory: builtin engine: scope directory %s unusable: %w", dir, err)
|
|
}
|
|
} else if _, err := os.Stat(dir); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, false, nil
|
|
}
|
|
return nil, false, fmt.Errorf("memory: builtin engine: scope directory %s unusable: %w", dir, err)
|
|
}
|
|
|
|
live, err := builtin.NewStore(filepath.Join(dir, liveFileName), e.maxFileBytes)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
history, err := builtin.NewStore(filepath.Join(dir, historyFileName), historyMaxFileBytes)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
stores := &scopeStores{live: live, history: history}
|
|
e.stores[key] = stores
|
|
return stores, true, nil
|
|
}
|
|
|
|
// scopeDirName maps a scope to a filesystem-safe, collision-resistant
|
|
// directory name.
|
|
func scopeDirName(scope domainmemory.Scope) string {
|
|
sum := sha256.Sum256([]byte(scope.Key()))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// sectionFor returns the section a record's Kind maps to, validated the same
|
|
// way builtin validates any section name -- so an invalid Kind fails with the
|
|
// error shape the memory tool already returns for an invalid section, not a
|
|
// new one.
|
|
func sectionFor(kind string) (string, error) {
|
|
if kind == "" {
|
|
kind = defaultSectionName
|
|
}
|
|
if err := builtin.ValidateSectionName(kind); err != nil {
|
|
return "", err
|
|
}
|
|
return kind, nil
|
|
}
|
|
|
|
// scanContent rejects content with a block-level scanner hit and logs
|
|
// warn-level hits.
|
|
func (e *BuiltinEngine) scanContent(content string) error {
|
|
result := contentScanner.ScanContent(content)
|
|
switch result.Level {
|
|
case domainmemory.ThreatBlock:
|
|
return fmt.Errorf("memory: builtin engine: refusing to persist content: %s", result.Message)
|
|
case domainmemory.ThreatWarn:
|
|
if e.log != nil {
|
|
e.log.Warn("memory: stored content matched a sensitive-data pattern",
|
|
"pattern", result.Pattern, "reason", result.Message)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// storeContent returns content as it will be stored: trimmed, and rejected
|
|
// when empty.
|
|
func storeContent(content string) (string, error) {
|
|
trimmed := strings.TrimSpace(content)
|
|
if trimmed == "" {
|
|
return "", errors.New("memory: builtin engine: content must not be empty")
|
|
}
|
|
return trimmed, nil
|
|
}
|
|
|
|
// findBlock returns the live block addressed by id.
|
|
func findBlock(store *builtin.Store, id domainmemory.RecordID) (parsedBlock, bool) {
|
|
for _, block := range parseBlocks(store.Render()) {
|
|
if block.marker.ID == string(id) {
|
|
return block, true
|
|
}
|
|
}
|
|
return parsedBlock{}, false
|
|
}
|
|
|
|
func notFound(scope domainmemory.Scope, id domainmemory.RecordID) error {
|
|
return fmt.Errorf("memory: builtin engine: %w: id %q in scope %s", domainmemory.ErrNotFound, id, scope)
|
|
}
|
|
|
|
// ── Store ──────────────────────────────────────────────────────────────
|
|
|
|
// Create records one new memory: the id is assigned once, here, and is the
|
|
// only address the record ever has.
|
|
func (e *BuiltinEngine) Create(_ context.Context, in domainmemory.NewRecord) (domainmemory.Record, error) {
|
|
if err := in.Validate(); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
section, err := sectionFor(in.Kind)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if err := e.scanContent(in.Content); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
content, err := storeContent(in.Content)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
stores, _, err := e.scopeStoresFor(in.Scope, true)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
|
|
now := e.now()
|
|
marker := markerData{
|
|
ID: uuid.NewString(),
|
|
Revision: 1,
|
|
Kind: section,
|
|
Author: in.Author,
|
|
OriginUser: string(in.OriginUser),
|
|
Source: in.Source,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
block, err := renderBlock(marker, content)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if _, err := stores.live.Add(section, block); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
return recordOf(marker, in.Scope, content), nil
|
|
}
|
|
|
|
func (e *BuiltinEngine) Get(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) (domainmemory.Record, error) {
|
|
if err := scope.Validate(); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if id == "" {
|
|
return domainmemory.Record{}, errors.New("memory: builtin engine: id must not be empty")
|
|
}
|
|
stores, ok, err := e.scopeStoresFor(scope, false)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if !ok {
|
|
return domainmemory.Record{}, notFound(scope, id)
|
|
}
|
|
block, ok := findBlock(stores.live, id)
|
|
if !ok {
|
|
return domainmemory.Record{}, notFound(scope, id)
|
|
}
|
|
return block.record(scope), nil
|
|
}
|
|
|
|
// List returns one scope's heads, most recently updated first.
|
|
func (e *BuiltinEngine) List(_ context.Context, scope domainmemory.Scope) ([]domainmemory.Record, error) {
|
|
if err := scope.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
stores, ok, err := e.scopeStoresFor(scope, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
blocks := parseBlocks(stores.live.Render())
|
|
records := make([]domainmemory.Record, 0, len(blocks))
|
|
for _, block := range blocks {
|
|
records = append(records, block.record(scope))
|
|
}
|
|
return orderHeads(records), nil
|
|
}
|
|
|
|
// orderHeads returns heads most recently updated first, ties in creation
|
|
// order.
|
|
func orderHeads(records []domainmemory.Record) []domainmemory.Record {
|
|
slices.Reverse(records)
|
|
slices.SortStableFunc(records, func(a, b domainmemory.Record) int {
|
|
return b.UpdatedAt.Compare(a.UpdatedAt)
|
|
})
|
|
return records
|
|
}
|
|
|
|
// Query returns the heads of exactly the scopes q names -- the caller's read
|
|
// set, never widened here -- grouped in the order those scopes were given and
|
|
// bounded by q.Limit across the whole result.
|
|
func (e *BuiltinEngine) Query(ctx context.Context, q domainmemory.Query) ([]domainmemory.Record, error) {
|
|
if err := q.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
limit := q.Limit
|
|
if limit == 0 {
|
|
limit = defaultQueryLimit
|
|
}
|
|
var out []domainmemory.Record
|
|
for _, scope := range q.Scopes {
|
|
heads, err := e.List(ctx, scope)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, record := range heads {
|
|
if q.Text != "" && !strings.Contains(record.Content, q.Text) {
|
|
continue
|
|
}
|
|
out = append(out, record)
|
|
if len(out) == limit {
|
|
return out, nil
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Update supersedes a record's content in place and returns the new state.
|
|
//
|
|
// A record may not move between scopes, which is why RecordUpdate carries no
|
|
// scope to change: it is addressed by the scope it is already in.
|
|
func (e *BuiltinEngine) Update(_ context.Context, in domainmemory.RecordUpdate) (domainmemory.Record, error) {
|
|
if err := in.Validate(); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
// Scanned before anything is written: a refused update must leave both
|
|
// the record and its history untouched.
|
|
if err := e.scanContent(in.Content); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
content, err := storeContent(in.Content)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
stores, ok, err := e.scopeStoresFor(in.Scope, false)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if !ok {
|
|
return domainmemory.Record{}, notFound(in.Scope, in.ID)
|
|
}
|
|
|
|
// Reload from disk before checking Expected.
|
|
if err := stores.live.Reload(); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
current, ok := findBlock(stores.live, in.ID)
|
|
if !ok {
|
|
return domainmemory.Record{}, notFound(in.Scope, in.ID)
|
|
}
|
|
// Expected is what makes a lost update detectable: the revision the
|
|
// caller last saw is the check, against the revision just read.
|
|
if in.Expected != 0 && in.Expected != current.marker.Revision {
|
|
return domainmemory.Record{}, fmt.Errorf(
|
|
"memory: builtin engine: %w: record %q is at revision %d, not %d",
|
|
domainmemory.ErrStaleRevision, in.ID, current.marker.Revision, in.Expected,
|
|
)
|
|
}
|
|
|
|
now := e.now()
|
|
next := current.marker
|
|
next.Revision = current.marker.Revision + 1
|
|
next.Author = in.Author
|
|
next.Source = in.Source
|
|
next.UpdatedAt = now
|
|
|
|
// Crash safety, in this order: the superseded state is durable in
|
|
// HISTORY.md before the live block is touched. A crash between the two
|
|
// leaves an extra history entry and an unchanged record -- recoverable,
|
|
if err := appendHistory(stores.history, current.marker, current.content(), now, false); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
block, err := renderBlock(next, content)
|
|
if err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
if _, err := stores.live.Replace(current.section, current.text, block, true); err != nil {
|
|
return domainmemory.Record{}, err
|
|
}
|
|
return recordOf(next, in.Scope, content), nil
|
|
}
|
|
|
|
// Forget removes a record's live state, retaining a deleted revision so its
|
|
// provenance outlives its content. Forgetting an id that is not there is not
|
|
// an error: the end state, id absent, already holds.
|
|
func (e *BuiltinEngine) Forget(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) error {
|
|
if err := scope.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if id == "" {
|
|
return errors.New("memory: builtin engine: id must not be empty")
|
|
}
|
|
stores, ok, err := e.scopeStoresFor(scope, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !ok {
|
|
return nil
|
|
}
|
|
current, ok := findBlock(stores.live, id)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
// The same crash-safe order as Update: the deleted revision is durable
|
|
// before the live block disappears, so the record's provenance is never
|
|
// the part that is lost.
|
|
if err := appendHistory(stores.history, current.marker, current.content(), e.now(), true); err != nil {
|
|
return err
|
|
}
|
|
_, err = stores.live.Remove(current.section, current.text, true)
|
|
return err
|
|
}
|
|
|
|
// Revisions returns a record's retained states, oldest first, including the
|
|
// deleted revision Forget records. A record that never existed is empty, not
|
|
// an error.
|
|
func (e *BuiltinEngine) Revisions(_ context.Context, scope domainmemory.Scope, id domainmemory.RecordID) ([]domainmemory.Revision, error) {
|
|
if err := scope.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
if id == "" {
|
|
return nil, errors.New("memory: builtin engine: id must not be empty")
|
|
}
|
|
stores, ok, err := e.scopeStoresFor(scope, false)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
return revisionsOf(stores.history, scope, id), nil
|
|
}
|