Files
archie/internal/app/statestore/recovery_postgres.go
T
sam 073959b4f9 feat(memory): scope chat notes and preserve deletions on restore (#2069)
Keep personal notes private by default and require review before publishing shared notes. Restore backups and replay forgotten notes in one transaction, and retain the installed release when an unattended rollback could bring back forgotten notes.
2026-11-21 13:20:31 +11:00

445 lines
17 KiB
Go

package statestore
import (
"context"
"crypto/sha256"
"encoding/hex"
json "encoding/json/v2"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/jackc/pgx/v5/stdlib"
"github.com/jmoiron/sqlx"
"github.com/samcharles93/archie-core/internal/infrastructure/postgres"
"github.com/samcharles93/archie-core/internal/platform/deletions"
"github.com/samcharles93/archie-core/modules/memory"
)
// runPostgresRecovery performs one recovery operation on the PostgreSQL
// database the configuration names. It connects as the migration role
// (migration_database_url, falling back to database_url): restoring and
// rolling back replace schema objects, which only the schema's owner may do.
// There is no automatic schema rollback: goose Down drops tables, so the way
// back from a migration is restoring the snapshot taken before it, which loses
// every write made after that snapshot.
func runPostgresRecovery(ctx context.Context, options RecoveryOptions) (string, error) {
cfg, err := bootConfig(ctx, options)
if err != nil {
return "", err
}
url := cfg.MigrationURL()
if url == "" {
return "", errors.New("database_url is required: the configuration names no PostgreSQL database")
}
switch options.Operation {
case RecoveryBackup:
if options.Out == "" {
return "", errors.New("backup requires -out")
}
if err := postgres.Backup(ctx, url, options.Out); err != nil {
return "", err
}
return "backed up the whole database to " + options.Out, nil
case RecoveryRestore:
return restorePostgres(ctx, url, options, cfg.StateDir)
case RecoveryValidate:
return validatePostgres(ctx, url, options)
case RecoveryRollback:
if options.Kind == "" {
return "", errors.New("rollback requires -kind")
}
return rollbackPostgres(ctx, url, options)
default:
return "", fmt.Errorf("unknown recovery command %q", options.Operation)
}
}
// restorePostgres replaces the database with the snapshot, refusing first when the
// data cannot show the restore is safe.
func restorePostgres(ctx context.Context, url string, options RecoveryOptions, stateDir string) (string, error) {
if options.From == "" {
return "", errors.New("restore requires -from")
}
dir, err := deletions.Dir(stateDir)
if err != nil {
return "", err
}
// One reading of the record inside the restore's window serves both the check
// and the replay, so what was proved safe is what is applied.
var recorded []memory.NoteKey
if err := postgres.Restore(ctx, url, options.From,
func(ctx context.Context, pool *pgxpool.Pool) error {
var err error
if recorded, err = memory.RecordedNotes(dir); err != nil {
return err
}
if options.Rollback {
return requireRollbackReplaysDeletions(ctx, pool, options.From, recorded)
}
return requireAccountedDeletions(ctx, pool, options.From, recorded, dir, options.AcceptUnrecordedDeletions)
},
func() (string, error) { return memory.DeletionStatements(recorded), nil }); err != nil {
return "", err
}
return fmt.Sprintf("restored the whole database from %s; every write made after the snapshot is gone, and %d recorded deletions were applied again; start the archie services again",
options.From, len(recorded)), nil
}
// fileDigest hashes a file without reading it into memory: snapshots of a real
// instance are far larger than a process should hold to checksum one.
func fileDigest(path string) (string, error) {
file, err := os.Open(path) //nolint:gosec // the snapshot the operator named.
if err != nil {
return "", fmt.Errorf("read the snapshot: %w", err)
}
defer func() { _ = file.Close() }()
hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
return "", fmt.Errorf("read the snapshot: %w", err)
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
// requireAccountedDeletions refuses a restore the record outside the database
// cannot account for, and refuses by default when it cannot say whether the data
// has seen deletions it does not know about. Freshness cannot be proved from this
// data alone: a snapshot and the record from the same backup match perfectly while
// later deletions are absent from both, so an operator's acknowledgement is the
// only way past a case where accounting is incomplete. It runs before anything is
// replaced.
func requireAccountedDeletions(ctx context.Context, pool *pgxpool.Pool, from string, recorded []memory.NoteKey, dir string, accepted bool) error {
facts, err := gatherRestoreFacts(ctx, pool, from, recorded, dir)
if err != nil {
return err
}
if accepted {
return acknowledge(ctx, from, dir, len(facts.unaccounted))
}
return refuseUnaccounted(facts, from, deletions.Path(dir))
}
// restoreFacts is what the check can learn before anything is replaced: what the
// target and the snapshot witnessed, whether the record outside the database is
// still there, and which witnesses it does not list.
type restoreFacts struct {
live []memory.Witness
snapshot []memory.Witness
ledgerExists bool
unaccounted []memory.Witness
}
// gatherRestoreFacts reads the witnesses from the target database and the
// snapshot, and compares them with the record outside the database.
func gatherRestoreFacts(ctx context.Context, pool *pgxpool.Pool, from string, recorded []memory.NoteKey, dir string) (restoreFacts, error) {
db := sqlx.NewDb(stdlib.OpenDBFromPool(pool), "pgx")
var facts restoreFacts
records, err := memory.HasWitnessTable(ctx, db)
if err != nil {
return restoreFacts{}, err
}
// A database that has never recorded a deletion has no witness table to read.
if records {
if facts.live, err = memory.Witnesses(ctx, db); err != nil {
return restoreFacts{}, fmt.Errorf("read the committed deletions: %w", err)
}
}
if facts.snapshot, _, err = snapshotWitnesses(ctx, from); err != nil {
return restoreFacts{}, err
}
facts.unaccounted = memory.Unaccounted(slices.Concat(facts.live, facts.snapshot), recorded)
if facts.ledgerExists, err = deletions.NewLog(deletions.Path(dir)).Exists(); err != nil {
return restoreFacts{}, err
}
return facts, nil
}
// refuseUnaccounted refuses a restore the record outside the database cannot
// account for. Nothing it reads can prove the record is current, so every doubt
// it cannot settle is a refusal, and an operator's acknowledgement is the one way
// past one.
func refuseUnaccounted(facts restoreFacts, from, recordPath string) error {
// With no record there is nothing to compare, and every deletion this
// instance ever made is unaccounted for.
if !facts.ledgerExists {
return fmt.Errorf("%w: the record outside the database, %s, is missing, so nothing here can show what this data has deleted. Restore the current record, or pass -accept-unrecorded-deletions to accept that forgotten notes may return",
ErrUnaccountedDeletions, recordPath)
}
// The target's own witnesses are the only evidence here about this data's
// history that does not come from the snapshot. With none, nothing can show
// what was deleted after the snapshot was taken: a fresh target and a target
// that has genuinely never deleted a note look the same, so the operator has
// to say which it is.
if len(facts.live) == 0 {
return fmt.Errorf("%w: this database has not recorded a deletion of its own, so nothing here can show what was deleted after %s was taken, and forgotten notes may return. Restore the record that belongs with the snapshot, or pass -accept-unrecorded-deletions to accept that risk",
ErrUnaccountedDeletions, from)
}
if len(facts.unaccounted) > 0 {
return fmt.Errorf("%w: the record outside the database does not list %d deletion(s) this data has seen, so a restore would bring forgotten notes back. Restore the current record, or pass -accept-unrecorded-deletions to accept that risk",
ErrUnaccountedDeletions, len(facts.unaccounted))
}
return nil
}
// rollbackFacts is what the update transaction's rollback compares: the notes its
// own snapshot holds, the notes the database holds now, and the deletions the
// record lists.
type rollbackFacts struct {
snapshotHasNotes bool
snapshot []memory.NoteKey
live []memory.NoteKey
recorded []memory.NoteKey
}
// requireRollbackReplaysDeletions proves a rollback safe from the data alone. The
// snapshot was taken by this host minutes earlier, so every note it holds and the
// database no longer has was deleted since, and applying those deletions to the
// snapshot is exactly what stops the rollback bringing them back. That proof does
// not rest on the witness table, which a migration may have emptied, so it holds
// with no witnesses at all; what it does need is every one of those deletions in
// the record, because the record is what the restore replays.
func requireRollbackReplaysDeletions(ctx context.Context, pool *pgxpool.Pool, from string, recorded []memory.NoteKey) error {
db := sqlx.NewDb(stdlib.OpenDBFromPool(pool), "pgx")
facts := rollbackFacts{recorded: recorded}
var err error
if facts.snapshotHasNotes, err = postgres.SnapshotHasTable(ctx, from, "memory.notes"); err != nil {
return err
}
if facts.snapshotHasNotes {
if facts.snapshot, err = snapshotNoteKeys(ctx, from); err != nil {
return err
}
}
if facts.live, err = memory.LiveNoteKeys(ctx, db); err != nil {
return err
}
return refuseUnsafeRollback(facts, from)
}
// refuseUnsafeRollback refuses a rollback the data cannot show to be safe.
func refuseUnsafeRollback(facts rollbackFacts, from string) error {
// A snapshot from before the notes table existed would replace a database that
// has notes with one that has none, and holds nothing to compare.
if !facts.snapshotHasNotes {
if len(facts.live) > 0 {
return fmt.Errorf("%w: %s holds no notes table and this database holds %d note(s), so the rollback would drop them with nothing to show what it did",
deletions.ErrUnsafeRollback, from, len(facts.live))
}
return nil
}
live := make(map[memory.NoteKey]bool, len(facts.live))
for _, key := range facts.live {
live[key] = true
}
recorded := make(map[memory.NoteKey]bool, len(facts.recorded))
for _, key := range facts.recorded {
recorded[key] = true
}
unrecorded := 0
for _, key := range facts.snapshot {
if live[key] || recorded[key] {
continue
}
unrecorded++
}
if unrecorded > 0 {
return fmt.Errorf("%w: %s holds %d note(s) this database no longer has and the record does not list, so a rollback would bring them back",
deletions.ErrUnsafeRollback, from, unrecorded)
}
return nil
}
// snapshotNoteKeys reads the notes an archive holds. Only the two columns a note
// is identified by are read, so a snapshot from before the memory schema was
// scoped can still be compared.
func snapshotNoteKeys(ctx context.Context, from string) ([]memory.NoteKey, error) {
columns, rows, err := postgres.SnapshotRows(ctx, from, "memory.notes")
if err != nil {
return nil, err
}
idAt, ok := postgres.Column(columns, "id")
if !ok {
return nil, fmt.Errorf("postgres: snapshot %s: the notes table has no id", from)
}
orgAt, ok := postgres.Column(columns, "org_id")
if !ok {
return nil, fmt.Errorf("postgres: snapshot %s: the notes table has no organisation", from)
}
out := make([]memory.NoteKey, 0, len(rows))
for _, row := range rows {
if len(row) <= max(idAt, orgAt) {
return nil, fmt.Errorf("postgres: snapshot %s: a note is missing a column", from)
}
id, err := uuid.Parse(strings.TrimSpace(row[idAt]))
if err != nil {
return nil, fmt.Errorf("postgres: snapshot %s: read a note: %w", from, err)
}
out = append(out, memory.NoteKey{OrgID: strings.TrimSpace(row[orgAt]), ID: id})
}
return out, nil
}
// acknowledge records that an operator chose to restore data the record cannot
// account for. The acknowledgement is written beside the record rather than
// instead of anything, so the evidence of what was lost survives.
func acknowledge(ctx context.Context, from, dir string, unaccounted int) error {
digest, err := fileDigest(from)
if err != nil {
return err
}
line, err := json.Marshal(map[string]any{
"acknowledged_at": time.Now().UTC(),
"snapshot": filepath.Base(from),
"snapshot_sha256": digest,
"unaccounted_deletions": unaccounted,
"consequence": "forgotten notes may return",
})
if err != nil {
return err
}
path := filepath.Join(dir, "deletions-acknowledged.jsonl")
if err := os.MkdirAll(dir, 0o700); err != nil {
return fmt.Errorf("record the acknowledgement: %w", err)
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) //nolint:gosec // the state directory.
if err != nil {
return fmt.Errorf("record the acknowledgement: %w", err)
}
defer func() { _ = file.Close() }()
if _, err := file.Write(append(line, '\n')); err != nil {
return fmt.Errorf("record the acknowledgement: %w", err)
}
if err := file.Sync(); err != nil {
return fmt.Errorf("record the acknowledgement: %w", err)
}
// The acknowledgement is evidence, so its name and the directory holding it
// have to be as durable as the line itself.
return deletions.SyncFileDirs(path)
}
// ErrUnaccountedDeletions is returned when a restore cannot account for deletions
// the data has seen.
var ErrUnaccountedDeletions = errors.New("the record of deletions does not account for this data")
// snapshotWitnesses reads the deletions a snapshot's own database witnessed. A
// snapshot from before the witness existed reports false: it never recorded any.
func snapshotWitnesses(ctx context.Context, from string) ([]memory.Witness, bool, error) {
const table = "memory.deletion_witness"
holds, err := postgres.SnapshotHasTable(ctx, from, table)
if err != nil {
return nil, false, err
}
if !holds {
return nil, false, nil
}
columns, rows, err := postgres.SnapshotRows(ctx, from, table)
if err != nil {
return nil, false, err
}
noteAt, ok := postgres.Column(columns, "note_id")
if !ok {
return nil, false, fmt.Errorf("postgres: snapshot %s: the deletion witness has no note id", from)
}
orgAt, ok := postgres.Column(columns, "org_id")
if !ok {
return nil, false, fmt.Errorf("postgres: snapshot %s: the deletion witness has no organisation", from)
}
out := make([]memory.Witness, 0, len(rows))
for _, row := range rows {
if len(row) <= max(noteAt, orgAt) {
return nil, false, fmt.Errorf("postgres: snapshot %s: a deletion witness is missing a column", from)
}
id, err := uuid.Parse(strings.TrimSpace(row[noteAt]))
if err != nil {
return nil, false, fmt.Errorf("postgres: snapshot %s: read a deletion witness: %w", from, err)
}
out = append(out, memory.Witness{NoteID: id, OrgID: strings.TrimSpace(row[orgAt])})
}
return out, true, nil
}
// openRecoveryPool opens the database without migrating it: recovery inspects
// or repairs the schema it finds, and refuses one newer than this binary.
func openRecoveryPool(ctx context.Context, url string) (*pgxpool.Pool, int64, error) {
pool, err := postgres.Open(ctx, url, postgres.AppStateStore+"-recovery")
if err != nil {
return nil, 0, err
}
version, err := postgres.SchemaVersion(ctx, pool)
if err == nil {
err = postgres.CheckSchemaVersion(version)
}
if err != nil {
pool.Close()
return nil, 0, err
}
return pool, version, nil
}
// validatePostgres answers whether archied would start against the database:
// the stored resources pass the writer's validation and boot's gate. An
// unmigrated database holds nothing to check; the State Store migrates and
// seeds it on its next start, so only boot's config gate applies.
func validatePostgres(ctx context.Context, url string, options RecoveryOptions) (string, error) {
pool, version, err := openRecoveryPool(ctx, url)
if err != nil {
return "", err
}
defer pool.Close()
server, err := openStateStoreControlPlane(postgres.NewResources(pool))
if err != nil {
return "", err
}
checked, err := checkBootGate(ctx, server, version > 0, options)
if err != nil {
return "", err
}
return fmt.Sprintf("the database is valid at schema version %d; %d stored resources validate", version, checked), nil
}
// rollbackPostgres replays an earlier revision holding the State Store's
// ownership claim, so it refuses while a State Store serves and none can
// start until it finishes.
func rollbackPostgres(ctx context.Context, url string, options RecoveryOptions) (string, error) {
pool, _, err := openRecoveryPool(ctx, url)
if err != nil {
return "", err
}
defer pool.Close()
claim, err := postgres.AcquireOwnership(ctx, pool, postgres.OwnerStateStore)
if err != nil {
return "", fmt.Errorf("rollback refused, stop the State Store first: %w", err)
}
defer func() { _ = claim.Release(context.WithoutCancel(ctx)) }()
return replayRevision(ctx, postgres.NewResources(pool), options)
}
// SchemaVersion reads the database's applied migration without migrating or
// refusing a schema newer than this binary, so an older binary can tell that
// a newer one migrated it.
func SchemaVersion(ctx context.Context, options RecoveryOptions) (int64, error) {
cfg, err := bootConfig(ctx, options)
if err != nil {
return 0, err
}
if cfg.DatabaseURL == "" {
return 0, errors.New("database_url is required: the configuration names no PostgreSQL database")
}
// The updater asks this while the services serve, so it reads the version
// through the runtime role: SELECT on goose's table is all it needs.
pool, err := postgres.Open(ctx, cfg.DatabaseURL, postgres.AppStateStore+"-schema")
if err != nil {
return 0, err
}
defer pool.Close()
return postgres.SchemaVersion(ctx, pool)
}