Files
archie/modules/memory/notes.go
T
sam 073959b4f9 feat(memory): scope chat notes and preserve deletions on restore (#2069)
Keep personal notes private by default and require review before publishing shared notes. Restore backups and replay forgotten notes in one transaction, and retain the installed release when an unattended rollback could bring back forgotten notes.
2026-11-21 13:20:31 +11:00

623 lines
22 KiB
Go

package memory
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"strings"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/internal/platform/deletions"
"github.com/samcharles93/archie-core/modules/memory/internal/store"
)
// maxNotes bounds the facts a conversation is given, so memory can't crowd
// out the conversation itself.
const maxNotes = 200
// notesTable names the table a recorded deletion applies to.
const notesTable = "memory.notes"
// DeletionsRecordedSince is the schema version from which this binary records
// note deletions outside the database. A restore onto a database at or above it
// must find that record: without it, deleted notes come back.
const DeletionsRecordedSince int64 = 41
// Class is one of the four memory classes. Each is read, written and retained
// on its own terms: a working context belongs to a session, episodic history and
// enterprise retrieval to a collection, and a durable fact to a person or to the
// org.
type Class string
const (
// ClassWorkingContext is the state of one session and turn.
ClassWorkingContext Class = "working_context"
// ClassEpisodicHistory is what earlier runs of a collection recall.
ClassEpisodicHistory Class = "episodic_history"
// ClassEnterpriseRetrieval is a retrieved enterprise document or chunk.
ClassEnterpriseRetrieval Class = "enterprise_retrieval"
// ClassDurableFact is what a person asked Archie to remember.
ClassDurableFact Class = "durable_fact"
)
// Classes returns every memory class.
func Classes() []Class {
return []Class{ClassWorkingContext, ClassEpisodicHistory, ClassEnterpriseRetrieval, ClassDurableFact}
}
// Validate reports whether c is a memory class.
func (c Class) Validate() error {
switch c {
case ClassWorkingContext, ClassEpisodicHistory, ClassEnterpriseRetrieval, ClassDurableFact:
return nil
}
return fmt.Errorf("%w: %q", ErrInvalidNote, c)
}
// Scope is who a note is readable by.
type Scope string
const (
// ScopePerson is readable by the owner alone.
ScopePerson Scope = "person"
// ScopeShared is readable by the org once an administrator publishes it.
ScopeShared Scope = "shared"
)
// State is whether a shared note is readable yet.
type State string
const (
// StatePending is a proposal only an administrator can see.
StatePending State = "pending"
// StatePublished is readable within its scope.
StatePublished State = "published"
)
// ErrInvalidNote is returned for a note that cannot be stored, such as one with
// no text, or one that carries a credential or an instruction that would
// override a security control.
var ErrInvalidNote = errors.New("invalid note")
// ErrRecordIncomplete is returned when a deletion committed but the record of it
// could not be written yet. It says exactly what happened, because a model
// relaying a bare failure would tell the operator the note is still there.
var ErrRecordIncomplete = errors.New("the note was deleted and the record of its deletion is pending; it is retried until it is written")
// Note is one remembered item.
type Note struct {
ID uuid.UUID `json:"id"`
Class Class `json:"class"`
Scope Scope `json:"scope"`
State State `json:"state"`
Purpose string `json:"purpose"`
Text string `json:"text"`
Owner string `json:"owner_id,omitempty"`
SharedBy string `json:"shared_by,omitempty"`
Version int `json:"version"`
Disabled bool `json:"disabled"`
CreatedAt time.Time `json:"created_at"`
}
// Witness is one committed deletion the record has to account for.
type Witness struct {
NoteID uuid.UUID `json:"note_id"`
OrgID string `json:"org_id"`
}
// NoteKey identifies one note by the organisation that owns it and its id, which
// is what a restore compares between a snapshot and the data it replaces.
type NoteKey struct {
OrgID string `db:"org_id" json:"org_id"`
ID uuid.UUID `db:"id" json:"id"`
}
// HasTable reports whether this database holds a table. A database from before
// the memory schema holds none of them, and that is not a failure.
func HasTable(ctx context.Context, db *sqlx.DB, table string) (bool, error) {
var found *string
if err := db.GetContext(ctx, &found, `SELECT to_regclass($1)::text`, table); err != nil {
return false, fmt.Errorf("read %s: %w", table, err)
}
return found != nil, nil
}
// HasWitnessTable reports whether this database records committed deletions. A
// database that does not has never run the release that records them, which is
// what tells a fresh target apart from one whose record has a history.
func HasWitnessTable(ctx context.Context, db *sqlx.DB) (bool, error) {
return HasTable(ctx, db, "memory.deletion_witness")
}
// LiveNoteKeys returns the notes this database holds now.
func LiveNoteKeys(ctx context.Context, db *sqlx.DB) ([]NoteKey, error) {
present, err := HasTable(ctx, db, notesTable)
if err != nil || !present {
return nil, err
}
var out []NoteKey
if err := db.SelectContext(ctx, &out, `SELECT org_id, id FROM memory.notes`); err != nil {
return nil, fmt.Errorf("read the notes this database holds: %w", err)
}
return out, nil
}
// Witnesses returns the deletions this database has committed, oldest first.
func Witnesses(ctx context.Context, db *sqlx.DB) ([]Witness, error) {
rows, err := store.New(db).Witnesses(ctx)
if err != nil {
return nil, err
}
out := make([]Witness, 0, len(rows))
for _, r := range rows {
out = append(out, Witness{NoteID: r.NoteID, OrgID: r.OrgID})
}
return out, nil
}
// Version is one published version of a shared note.
type Version struct {
Version int `json:"version"`
Text string `json:"text"`
Purpose string `json:"purpose"`
PublisherID string `json:"publisher_id"`
PublishedAt time.Time `json:"published_at"`
}
// DeletionLog records a deletion outside the database, so a restore cannot
// bring back what was deleted after the snapshot was taken.
type DeletionLog interface {
Append(ctx context.Context, entry deletions.Entry) error
}
// noteStore is the storage a person's own notes need.
type noteStore interface {
AddNote(ctx context.Context, orgID string, n store.Note) (store.Note, error)
Owns(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error)
Notes(ctx context.Context, orgID, owner, class string, limit int) ([]store.Note, error)
Owned(ctx context.Context, orgID, owner, class string, limit int) ([]store.Note, error)
Correct(ctx context.Context, orgID, owner string, id uuid.UUID, text string) (store.Note, bool, error)
SetDisabled(ctx context.Context, orgID, owner string, id uuid.UUID, disabled bool) (bool, error)
CommitDeletion(ctx context.Context, orgID, owner string, id uuid.UUID) (bool, error)
}
// sharedStore is the storage the shared scope needs: proposals, publication
// and the versions a publication can be rolled back to.
type sharedStore interface {
SharedExists(ctx context.Context, orgID string, id uuid.UUID) (bool, error)
Pending(ctx context.Context, orgID string, limit int) ([]store.Note, error)
Shared(ctx context.Context, orgID, class string, limit int) ([]store.Note, error)
Publish(ctx context.Context, orgID string, id uuid.UUID, publisher string) (store.Note, bool, error)
Versions(ctx context.Context, noteID uuid.UUID) ([]store.Version, error)
Rollback(ctx context.Context, orgID string, id uuid.UUID, version int, publisher string) (store.Note, bool, error)
CommitSharedDeletion(ctx context.Context, orgID string, id uuid.UUID) (bool, error)
}
// Remember stores a fact in owner's scope. The owner comes from the turn being
// served, never from the model, and a fact that carries a credential or would
// override a security control is refused.
func (m *Module) Remember(ctx context.Context, owner, text string) (Note, error) {
owner = strings.TrimSpace(owner)
if owner == "" {
return Note{}, fmt.Errorf("%w: a fact belongs to a person", ErrInvalidNote)
}
text, err := minimised(text)
if err != nil {
return Note{}, err
}
row, err := m.notes.AddNote(ctx, m.orgID, store.Note{
ID: uuid.New(), Class: string(ClassDurableFact), Scope: string(ScopePerson),
State: string(StatePublished), OwnerID: owner, Purpose: purposeChat, Text: text, Version: 1,
})
if err != nil {
return Note{}, err
}
return noteOf(row), nil
}
// purposeChat is what a fact written from a conversation is for.
const purposeChat = "chat"
// Notes returns the notes of one class that owner may read: their own, and the
// ones an administrator published to the org.
func (m *Module) Notes(ctx context.Context, owner string, class Class) ([]Note, error) {
owner = strings.TrimSpace(owner)
if owner == "" {
return nil, fmt.Errorf("%w: a reader is needed to know what it may read", ErrInvalidNote)
}
if err := class.Validate(); err != nil {
return nil, err
}
rows, err := m.notes.Notes(ctx, m.orgID, owner, string(class), maxNotes)
if err != nil {
return nil, err
}
out := make([]Note, 0, len(rows))
for _, r := range rows {
out = append(out, noteOf(r))
}
return out, nil
}
// Mine returns every note of one class in owner's own scope, including the ones
// they disabled, so they can inspect and re-enable them. A conversation reads
// Notes, which leaves those out.
func (m *Module) Mine(ctx context.Context, owner string, class Class) ([]Note, error) {
owner = strings.TrimSpace(owner)
if owner == "" {
return nil, fmt.Errorf("%w: a reader is needed to know what it may read", ErrInvalidNote)
}
if err := class.Validate(); err != nil {
return nil, err
}
rows, err := m.notes.Owned(ctx, m.orgID, owner, string(class), maxNotes)
if err != nil {
return nil, err
}
out := make([]Note, 0, len(rows))
for _, r := range rows {
out = append(out, noteOf(r))
}
return out, nil
}
// Correct replaces the text of one of owner's own notes.
func (m *Module) Correct(ctx context.Context, owner string, id uuid.UUID, text string) (Note, bool, error) {
text, err := minimised(text)
if err != nil {
return Note{}, false, err
}
row, ok, err := m.notes.Correct(ctx, m.orgID, strings.TrimSpace(owner), id, text)
if err != nil || !ok {
return Note{}, ok, err
}
return noteOf(row), true, nil
}
// Disable stops one of owner's own notes being read, or resumes it.
func (m *Module) Disable(ctx context.Context, owner string, id uuid.UUID, disabled bool) (bool, error) {
return m.notes.SetDisabled(ctx, m.orgID, strings.TrimSpace(owner), id, disabled)
}
// Forget deletes one of owner's own notes. The deletion is recorded outside the
// database before the row goes, so a failed recording leaves the note in place
// rather than removing one the next restore would bring back.
func (m *Module) Forget(ctx context.Context, owner string, id uuid.UUID) (bool, error) {
owner = strings.TrimSpace(owner)
owns, err := m.notes.Owns(ctx, m.orgID, owner, id)
if err != nil || !owns {
return false, err
}
ok, err := m.notes.CommitDeletion(ctx, m.orgID, owner, id)
if err != nil || !ok {
return ok, err
}
return true, m.recordDeletion(ctx, id)
}
// Propose records a note a person wants the org to have. Only administrators
// read it until one of them publishes it.
func (m *Module) Propose(ctx context.Context, owner, text string) (Note, error) {
owner = strings.TrimSpace(owner)
if owner == "" {
return Note{}, fmt.Errorf("%w: a proposal comes from a person", ErrInvalidNote)
}
text, err := minimised(text)
if err != nil {
return Note{}, err
}
row, err := m.notes.AddNote(ctx, m.orgID, store.Note{
ID: uuid.New(), Class: string(ClassDurableFact), Scope: string(ScopeShared),
State: string(StatePending), SharedBy: owner, Purpose: purposeChat, Text: text,
})
if err != nil {
return Note{}, err
}
return noteOf(row), nil
}
// Pending returns the shared notes waiting for an administrator.
func (m *Module) Pending(ctx context.Context) ([]Note, error) {
rows, err := m.shared.Pending(ctx, m.orgID, maxNotes)
if err != nil {
return nil, err
}
out := make([]Note, 0, len(rows))
for _, r := range rows {
out = append(out, noteOf(r))
}
return out, nil
}
// Shared returns the notes an administrator published to the org.
func (m *Module) Shared(ctx context.Context, class Class) ([]Note, error) {
if err := class.Validate(); err != nil {
return nil, err
}
rows, err := m.shared.Shared(ctx, m.orgID, string(class), maxNotes)
if err != nil {
return nil, err
}
out := make([]Note, 0, len(rows))
for _, r := range rows {
out = append(out, noteOf(r))
}
return out, nil
}
// Publish stores a note in the shared scope for everyone, recording the
// administrator who published it as its first version.
func (m *Module) Publish(ctx context.Context, publisher, purpose, text string) (Note, error) {
publisher = strings.TrimSpace(publisher)
if publisher == "" {
return Note{}, fmt.Errorf("%w: a published note names its publisher", ErrInvalidNote)
}
text, err := minimised(text)
if err != nil {
return Note{}, err
}
if strings.TrimSpace(purpose) == "" {
purpose = purposeChat
}
added, err := m.notes.AddNote(ctx, m.orgID, store.Note{
ID: uuid.New(), Class: string(ClassDurableFact), Scope: string(ScopeShared),
State: string(StatePending), SharedBy: publisher, Purpose: purpose, Text: text,
})
if err != nil {
return Note{}, err
}
row, ok, err := m.shared.Publish(ctx, m.orgID, added.ID, publisher)
if err != nil {
return Note{}, err
}
if !ok {
return Note{}, fmt.Errorf("%w: the note is no longer a proposal", ErrInvalidNote)
}
return noteOf(row), nil
}
// Approve publishes a pending proposal under the administrator approving it.
func (m *Module) Approve(ctx context.Context, publisher string, id uuid.UUID) (Note, bool, error) {
row, ok, err := m.shared.Publish(ctx, m.orgID, id, strings.TrimSpace(publisher))
if err != nil || !ok {
return Note{}, ok, err
}
return noteOf(row), true, nil
}
// Revoke removes a shared note, published or pending. The deletion is recorded
// before the row goes, as in Forget.
func (m *Module) Revoke(ctx context.Context, id uuid.UUID) (bool, error) {
exists, err := m.shared.SharedExists(ctx, m.orgID, id)
if err != nil || !exists {
return false, err
}
ok, err := m.shared.CommitSharedDeletion(ctx, m.orgID, id)
if err != nil || !ok {
return ok, err
}
return true, m.recordDeletion(ctx, id)
}
// Versions returns every published version of a shared note, newest first.
func (m *Module) Versions(ctx context.Context, id uuid.UUID) ([]Version, error) {
rows, err := m.shared.Versions(ctx, id)
if err != nil {
return nil, err
}
out := make([]Version, 0, len(rows))
for _, r := range rows {
out = append(out, Version{
Version: r.Version, Text: r.Text, Purpose: r.Purpose,
PublisherID: r.PublisherID, PublishedAt: r.PublishedAt,
})
}
return out, nil
}
// Rollback republishes an earlier version of a shared note as a new version.
func (m *Module) Rollback(ctx context.Context, publisher string, id uuid.UUID, version int) (Note, bool, error) {
row, ok, err := m.shared.Rollback(ctx, m.orgID, id, version, strings.TrimSpace(publisher))
if err != nil || !ok {
return Note{}, ok, err
}
return noteOf(row), true, nil
}
// recordDeletion appends the committed deletion to the log outside the database.
// A recording that fails is reported as ErrRecordIncomplete rather than hidden:
// the deletion has happened, and Reconcile completes its record from the witness
// at the next start.
func (m *Module) recordDeletion(ctx context.Context, id uuid.UUID) error {
if m.deleted == nil {
return fmt.Errorf("%w: no record of deletions is configured", ErrRecordIncomplete)
}
if err := m.deleted.Append(ctx, deletions.Entry{
Table: notesTable, OrgID: m.orgID, ID: id, At: time.Now().UTC(),
}); err != nil {
m.failures.Add(1)
return fmt.Errorf("%w: %w", ErrRecordIncomplete, err)
}
return nil
}
// RecordPending reports whether a committed deletion is still waiting for its
// entry in the record outside the database.
func (m *Module) RecordPending() bool { return m.failures.Load() > m.cleared.Load() }
// FailureMark is the failure count to hand back to MarkRecordComplete once a
// reconcile has finished, so a failure recorded while it ran is not reported as
// resolved by it.
func (m *Module) FailureMark() uint64 { return m.failures.Load() }
// MarkRecordComplete records that a reconcile which began at mark resolved every
// failure up to it. Failures after mark stay pending.
func (m *Module) MarkRecordComplete(mark uint64) {
for {
cleared := m.cleared.Load()
if cleared >= mark || m.cleared.CompareAndSwap(cleared, mark) {
return
}
}
}
// Reconcile completes the record: every committed deletion the record does not
// list is appended, so a crash between a commit and its entry cannot leave a
// deletion unaccounted for. It runs at boot and is safe to repeat.
func Reconcile(ctx context.Context, db *sqlx.DB, stateDir string) (int, error) {
witnesses, err := Witnesses(ctx, db)
if err != nil {
return 0, err
}
return reconcile(ctx, witnesses, deletions.NewLog(deletions.Path(stateDir)))
}
// reconcile appends an entry for every committed deletion the record does not
// list yet, so a crash between a commit and its entry cannot leave a deletion
// unaccounted for. It is idempotent: a second pass over the same deletions adds
// nothing.
func reconcile(ctx context.Context, witnesses []Witness, log *deletions.Log) (int, error) {
entries, err := log.Entries()
if err != nil {
return 0, err
}
recorded := make(map[uuid.UUID]bool, len(entries))
for _, e := range entries {
recorded[e.ID] = true
}
added := 0
for _, w := range witnesses {
if recorded[w.NoteID] {
continue
}
if err := log.Append(ctx, deletions.Entry{
Table: notesTable, OrgID: w.OrgID, ID: w.NoteID, At: time.Now().UTC(),
}); err != nil {
return added, err
}
recorded[w.NoteID] = true
added++
}
return added, nil
}
// Unaccounted returns the committed deletions the record outside the database
// does not list. A restore refuses to proceed while any of them is missing: the
// record would replay less than the database says happened. The caller reads the
// record once and passes the same entries it will replay.
func Unaccounted(witnesses []Witness, recorded []NoteKey) []Witness {
byNote := make(map[uuid.UUID]string, len(recorded))
for _, r := range recorded {
byNote[r.ID] = r.OrgID
}
var missing []Witness
for _, w := range witnesses {
if org, ok := byNote[w.NoteID]; !ok || org != w.OrgID {
missing = append(missing, w)
}
}
return missing
}
// ReapplyDeletions applies the deletions recorded outside the database to a
// restored one, so a snapshot cannot restore a note deleted after it was taken.
// stateDir is the resolved state directory. It runs after a restore and again at
// boot: a restore that replaced the database and then failed to apply the record
// leaves the notes it recorded in place, and none of them is served.
func ReapplyDeletions(ctx context.Context, db *sqlx.DB, stateDir string) (int, error) {
return reapplyDeletions(ctx, store.New(db), stateDir)
}
// noteDeleter removes notes by id from one organisation.
type noteDeleter interface {
DeleteNotes(ctx context.Context, orgID string, ids []uuid.UUID) (int, error)
}
// recordedNotes reads the note deletions the record holds. Both the replay that
// runs in Go and the statements a restore sends inside its transaction come from
// here, so the two can never disagree about which entries apply.
func recordedNotes(stateDir string) ([]NoteKey, error) {
entries, err := deletions.NewLog(deletions.Path(stateDir)).Entries()
if err != nil {
return nil, err
}
out := make([]NoteKey, 0, len(entries))
for _, e := range entries {
if e.Table == notesTable && e.ID != uuid.Nil && e.OrgID != "" {
out = append(out, NoteKey{OrgID: e.OrgID, ID: e.ID})
}
}
return out, nil
}
// RecordedNotes returns the note deletions the record outside the database holds.
func RecordedNotes(stateDir string) ([]NoteKey, error) { return recordedNotes(stateDir) }
// reapplyDeletions deletes what the record lists, grouped by the organisation each
// entry names, so a restore cannot put a deleted note back in front of anyone.
func reapplyDeletions(ctx context.Context, notes noteDeleter, stateDir string) (int, error) {
recorded, err := recordedNotes(stateDir)
if err != nil {
return 0, err
}
byOrg := map[string][]uuid.UUID{}
for _, r := range recorded {
byOrg[r.OrgID] = append(byOrg[r.OrgID], r.ID)
}
reapplied := 0
for _, org := range slices.Sorted(maps.Keys(byOrg)) {
n, err := notes.DeleteNotes(ctx, org, byOrg[org])
if err != nil {
return reapplied, err
}
reapplied += n
}
return reapplied, nil
}
// DeletionStatements renders recorded note deletions as SQL for a restore to run
// inside the transaction that replaced the database. The caller passes the same
// entries it checked the restore against, so what is applied and what was proved
// safe cannot drift apart. A snapshot from before the memory schema existed holds
// no notes table, so the statements do nothing there rather than failing the
// restore. Nothing the record holds is pasted in as SQL text: the organisation is
// rendered as its bytes, so no value in it can end a literal or the block.
func DeletionStatements(recorded []NoteKey) string {
if len(recorded) == 0 {
return ""
}
var b strings.Builder
b.WriteString("SET standard_conforming_strings = on;\n")
b.WriteString("DO $deletions$\nBEGIN\n")
for _, r := range recorded {
// The organisation is decoded from its bytes, so no text the record holds
// is pasted into the statement and nothing in it can end the block.
org := fmt.Sprintf("convert_from(decode('%x', 'hex'), 'UTF8')", r.OrgID)
b.WriteString(" IF to_regclass('memory.notes') IS NOT NULL THEN\n")
fmt.Fprintf(&b, " DELETE FROM memory.notes WHERE org_id = %s AND id = '%s';\n", org, r.ID)
b.WriteString(" END IF;\n")
// The witness is restored with the deletion it witnesses. The snapshot was
// taken before that deletion, so its own witness table cannot hold it, and a
// later restore would otherwise be unable to tell what this database has
// deleted once the record is lost.
b.WriteString(" IF to_regclass('memory.deletion_witness') IS NOT NULL THEN\n")
fmt.Fprintf(&b, " INSERT INTO memory.deletion_witness (note_id, org_id) VALUES ('%s', %s) ON CONFLICT (note_id) DO NOTHING;\n", r.ID, org)
b.WriteString(" END IF;\n")
}
b.WriteString("END\n$deletions$;\n")
return b.String()
}
func noteOf(r store.Note) Note {
return Note{
ID: r.ID, Class: Class(r.Class), Scope: Scope(r.Scope), State: State(r.State),
Purpose: r.Purpose, Text: r.Text, Owner: r.OwnerID, SharedBy: r.SharedBy,
Version: r.Version, Disabled: r.DisabledAt != nil, CreatedAt: r.CreatedAt,
}
}