Keep personal notes private by default and require review before publishing shared notes. Restore backups and replay forgotten notes in one transaction, and retain the installed release when an unattended rollback could bring back forgotten notes.
511 lines
18 KiB
Go
511 lines
18 KiB
Go
package postgres
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/pressly/goose/v3"
|
|
|
|
platformpg "github.com/samcharles93/archie-core/internal/platform/pg"
|
|
)
|
|
|
|
// ServiceOwners is every serving role that reads or writes the database. A
|
|
// destructive offline operation must hold all of them: one database backs
|
|
// every service, so the State Store being stopped alone proves nothing about
|
|
// the Gateway.
|
|
var ServiceOwners = []string{OwnerStateStore, OwnerGateway}
|
|
|
|
// ServeLockName is the advisory lock every serving process holds shared and a
|
|
// restore takes exclusively: several processes may serve at once, none may serve
|
|
// while a restore runs, and no restore may run while any serves.
|
|
const ServeLockName = platformpg.ServeLockName
|
|
|
|
// ErrSchemaTooNew is returned for a database migrated by a newer release than
|
|
// this binary. goose Down is not a rollback (it drops tables), so the only way
|
|
// back is restoring the snapshot taken before that migration.
|
|
var ErrSchemaTooNew = errors.New("postgres: database schema is newer than this binary supports")
|
|
|
|
// gooseTable is goose's version table; its presence is what marks an archive
|
|
// as a dump of an archie database rather than of anything pg_restore can read.
|
|
const gooseTable = "public.goose_db_version"
|
|
|
|
// SupportedSchemaVersion is the highest migration this binary embeds.
|
|
func SupportedSchemaVersion() int64 {
|
|
names, err := fs.Glob(Migrations(), "*.sql")
|
|
if err != nil {
|
|
// Unreachable: the pattern is well formed.
|
|
panic(err)
|
|
}
|
|
var highest int64
|
|
for _, name := range names {
|
|
if version, err := goose.NumericComponent(name); err == nil {
|
|
highest = max(highest, version)
|
|
}
|
|
}
|
|
return highest
|
|
}
|
|
|
|
// CheckSchemaVersion refuses a schema this binary cannot serve.
|
|
func CheckSchemaVersion(version int64) error {
|
|
if supported := SupportedSchemaVersion(); version > supported {
|
|
return fmt.Errorf("%w (database at %d, binary supports %d)", ErrSchemaTooNew, version, supported)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SchemaVersion reads the applied goose version without migrating. An
|
|
// unmigrated database is version 0.
|
|
func SchemaVersion(ctx context.Context, pool *pgxpool.Pool) (int64, error) {
|
|
// The table is looked up before it is named: PostgreSQL resolves a relation
|
|
// at parse time, so one query that mentions it fails on a database that has
|
|
// never been migrated.
|
|
var table *string
|
|
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.goose_db_version')::text`).Scan(&table); err != nil {
|
|
return 0, fmt.Errorf("postgres: read schema version: %w", err)
|
|
}
|
|
if table == nil {
|
|
return 0, nil
|
|
}
|
|
var version int64
|
|
if err := pool.QueryRow(ctx, `SELECT coalesce(max(version_id), 0) FROM public.goose_db_version WHERE is_applied`).Scan(&version); err != nil {
|
|
return 0, fmt.Errorf("postgres: read schema version: %w", err)
|
|
}
|
|
return version, nil
|
|
}
|
|
|
|
// AcquireAll claims every name, or none: on any failure the claims already
|
|
// taken are released before the error returns.
|
|
func AcquireAll(ctx context.Context, pool *pgxpool.Pool, names []string) (release func(context.Context), err error) {
|
|
held := make([]*Ownership, 0, len(names))
|
|
release = func(ctx context.Context) {
|
|
for _, o := range slices.Backward(held) {
|
|
_ = o.Release(ctx)
|
|
}
|
|
}
|
|
for _, name := range names {
|
|
o, err := AcquireOwnership(ctx, pool, name)
|
|
if err != nil {
|
|
release(ctx)
|
|
return nil, err
|
|
}
|
|
held = append(held, o)
|
|
}
|
|
return release, nil
|
|
}
|
|
|
|
// Backup writes a pg_dump archive to out, replacing it only once pg_restore
|
|
// can read the new archive.
|
|
func Backup(ctx context.Context, databaseURL, out string) error {
|
|
tmp, err := os.CreateTemp(filepath.Dir(out), filepath.Base(out)+".tmp*")
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: backup: %w", err)
|
|
}
|
|
scratch := tmp.Name()
|
|
_ = tmp.Close()
|
|
defer func() { _ = os.Remove(scratch) }()
|
|
|
|
dbname, env := libpqTarget(databaseURL)
|
|
if _, err := runTool(ctx, env, "pg_dump", dumpArgs(dbname, scratch)...); err != nil {
|
|
return fmt.Errorf("postgres: backup: %w", err)
|
|
}
|
|
if _, err := ValidateSnapshot(ctx, scratch); err != nil {
|
|
return fmt.Errorf("postgres: backup produced an unreadable snapshot: %w", err)
|
|
}
|
|
if err := os.Rename(scratch, out); err != nil {
|
|
return fmt.Errorf("postgres: backup: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ValidateSnapshot reads the archive's table of contents and returns the
|
|
// tables it holds. An archive without goose's version table is not a dump of
|
|
// an archie database.
|
|
func ValidateSnapshot(ctx context.Context, path string) ([]string, error) {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return nil, fmt.Errorf("postgres: snapshot: %w", err)
|
|
}
|
|
list, err := runTool(ctx, nil, "pg_restore", "--list", path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: snapshot %s: %w", path, err)
|
|
}
|
|
tables := snapshotTables(list)
|
|
if !slices.Contains(tables, gooseTable) {
|
|
return nil, fmt.Errorf("postgres: snapshot %s holds no %s; it is not an archie database dump", path, gooseTable)
|
|
}
|
|
return tables, nil
|
|
}
|
|
|
|
// SnapshotSchemaVersion reads the schema version the snapshot was taken at. A
|
|
// restore onto an empty database has no other way to know which release took it,
|
|
// and so whether the instance was one that records deletions outside it.
|
|
func SnapshotSchemaVersion(ctx context.Context, path string) (int64, error) {
|
|
if _, err := ValidateSnapshot(ctx, path); err != nil {
|
|
return 0, err
|
|
}
|
|
// pg_restore matches --table against the bare name, and the copy header then
|
|
// says which schema's table the rows belong to.
|
|
out, err := runTool(ctx, nil, "pg_restore", "--data-only", "--file=-", "--table=goose_db_version", path)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("postgres: snapshot %s: %w", path, err)
|
|
}
|
|
var version int64
|
|
inRows := false
|
|
for line := range strings.SplitSeq(out, "\n") {
|
|
if strings.HasPrefix(line, "COPY ") {
|
|
inRows = strings.Contains(line, gooseTable)
|
|
continue
|
|
}
|
|
if !inRows {
|
|
continue
|
|
}
|
|
if line == "\\." {
|
|
inRows = false
|
|
continue
|
|
}
|
|
// id, version_id, is_applied
|
|
fields := strings.Split(line, "\t")
|
|
if len(fields) < 3 || fields[2] != "t" {
|
|
continue
|
|
}
|
|
id, err := strconv.ParseInt(strings.TrimSpace(fields[1]), 10, 64)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("postgres: snapshot %s: read the schema version: %w", path, err)
|
|
}
|
|
version = max(version, id)
|
|
}
|
|
return version, nil
|
|
}
|
|
|
|
// SnapshotHasTable reports whether a snapshot holds a table. It is how a restore
|
|
// tells a snapshot from before a table existed from one with an empty table.
|
|
func SnapshotHasTable(ctx context.Context, path, table string) (bool, error) {
|
|
tables, err := ValidateSnapshot(ctx, path)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return slices.Contains(tables, table), nil
|
|
}
|
|
|
|
// SnapshotRows reads the data-only rows of one table from a snapshot, with the
|
|
// column names the COPY header gives, because the order an archive writes them in
|
|
// is not the order this binary reads the table in. A table the archive does not
|
|
// hold has no columns and no rows.
|
|
func SnapshotRows(ctx context.Context, path, table string) ([]string, [][]string, error) {
|
|
out, err := runTool(ctx, nil, "pg_restore", "--data-only", "--file=-", "--table="+bareTable(table), path)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("postgres: snapshot %s: %w", path, err)
|
|
}
|
|
var columns []string
|
|
var rows [][]string
|
|
inRows := false
|
|
for line := range strings.SplitSeq(out, "\n") {
|
|
if strings.HasPrefix(line, "COPY ") {
|
|
inRows = strings.Contains(line, table)
|
|
if inRows {
|
|
columns = copyColumns(line)
|
|
}
|
|
continue
|
|
}
|
|
if !inRows {
|
|
continue
|
|
}
|
|
if line == "\\." {
|
|
inRows = false
|
|
continue
|
|
}
|
|
rows = append(rows, strings.Split(line, "\t"))
|
|
}
|
|
return columns, rows, nil
|
|
}
|
|
|
|
// copyColumns reads the column names from a COPY header such as
|
|
// "COPY memory.notes (id, org_id) FROM stdin;".
|
|
func copyColumns(header string) []string {
|
|
open := strings.Index(header, "(")
|
|
shut := strings.LastIndex(header, ")")
|
|
if open < 0 || shut < open {
|
|
return nil
|
|
}
|
|
names := strings.Split(header[open+1:shut], ",")
|
|
for i, name := range names {
|
|
names[i] = strings.TrimSpace(name)
|
|
}
|
|
return names
|
|
}
|
|
|
|
// Column returns the index of a named column, and whether it is there.
|
|
func Column(columns []string, name string) (int, bool) {
|
|
at := slices.Index(columns, name)
|
|
return at, at >= 0
|
|
}
|
|
|
|
// bareTable drops the schema from a qualified table name, which is the form
|
|
// pg_restore matches.
|
|
func bareTable(table string) string {
|
|
if i := strings.LastIndex(table, "."); i >= 0 {
|
|
return table[i+1:]
|
|
}
|
|
return table
|
|
}
|
|
|
|
// Restore replaces the whole database with the snapshot at from, in one
|
|
// transaction: the tables the archive does not hold are dropped, the archive is
|
|
// applied, replay's statements run, and only then does it commit. A failure at
|
|
// any point, including an archive that ends early, reaches PostgreSQL as EOF
|
|
// before COMMIT, so the transaction aborts and the database is left as it was.
|
|
//
|
|
// It refuses unless it can claim every service role and the exclusive serve lock,
|
|
// and holds both for the whole window: before runs while the database is still
|
|
// the old one, so a refusal there leaves it untouched, and no serving process can
|
|
// commit a change between the check and the work that depends on it.
|
|
func Restore(ctx context.Context, databaseURL, from string, before func(context.Context, *pgxpool.Pool) error, replay func() (string, error)) error {
|
|
tables, err := ValidateSnapshot(ctx, from)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pool, err := Open(ctx, databaseURL, AppStateStore+"-restore")
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: restore cannot confirm every service is stopped: %w", err)
|
|
}
|
|
defer pool.Close()
|
|
release, err := AcquireAll(ctx, pool, ServiceOwners)
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: restore refused, stop every archie service first: %w", err)
|
|
}
|
|
defer release(context.WithoutCancel(ctx))
|
|
serve, err := claimServeExclusive(ctx, pool)
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: restore refused, stop every archie service first: %w", err)
|
|
}
|
|
defer serve(context.WithoutCancel(ctx))
|
|
|
|
if before != nil {
|
|
if err := before(ctx, pool); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
drops, err := dropStatementsNotIn(ctx, pool, tables)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// The caller's statements are read before anything is replaced: a record that
|
|
// cannot be read is a reason not to start, not a reason to commit without it.
|
|
var trailer string
|
|
if replay != nil {
|
|
if trailer, err = replay(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return applySnapshot(ctx, databaseURL, from, drops, trailer)
|
|
}
|
|
|
|
// applySnapshot replaces the database inside one transaction: it sends the drop
|
|
// statements, the archive and the caller's statements to psql, and commits only
|
|
// when the archive was read to its end and psql accepted everything. It drives
|
|
// psql and pg_restore from the PostgreSQL client tools, and tells psql to stop at
|
|
// the first error, so a statement that fails ends the session with the transaction
|
|
// still open, and PostgreSQL rolls it back.
|
|
func applySnapshot(ctx context.Context, databaseURL, from string, drops []string, trailer string) error {
|
|
dbname, env := libpqTarget(databaseURL)
|
|
dump := exec.CommandContext(ctx, "pg_restore", restoreArgs(from)...)
|
|
dump.Env = append(os.Environ(), env...)
|
|
var dumpStderr bytes.Buffer
|
|
dump.Stderr = &dumpStderr
|
|
dumpOut, err := dump.StdoutPipe()
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: restore: %w", err)
|
|
}
|
|
sql := exec.CommandContext(ctx, "psql", psqlArgs(dbname)...)
|
|
sql.Env = append(os.Environ(), env...)
|
|
sql.Stdout = io.Discard
|
|
var sqlErr bytes.Buffer
|
|
sql.Stderr = &sqlErr
|
|
stdin, err := sql.StdinPipe()
|
|
if err != nil {
|
|
return fmt.Errorf("postgres: restore: %w", err)
|
|
}
|
|
if err := sql.Start(); err != nil {
|
|
return fmt.Errorf("postgres: restore: %w", err)
|
|
}
|
|
if err := dump.Start(); err != nil {
|
|
_ = stdin.Close()
|
|
_ = sql.Wait()
|
|
return fmt.Errorf("postgres: restore: %w", err)
|
|
}
|
|
write := func(s string) error {
|
|
_, err := io.WriteString(stdin, s)
|
|
return err
|
|
}
|
|
body := "BEGIN;\n" + strings.Join(drops, "\n")
|
|
if len(drops) > 0 {
|
|
body += "\n"
|
|
}
|
|
bodyErr := write(body)
|
|
if bodyErr == nil {
|
|
_, bodyErr = io.Copy(stdin, dumpOut)
|
|
}
|
|
// psql exits on its first error, so a write or a copy that failed means nothing
|
|
// will read the archive again. The producer is stopped before it is waited on:
|
|
// a pg_restore blocked on a full pipe never returns on its own, and a truncated
|
|
// stream is what keeps the transaction from being committed.
|
|
if bodyErr != nil {
|
|
_ = dumpOut.Close()
|
|
if dump.Process != nil {
|
|
_ = dump.Process.Kill()
|
|
}
|
|
}
|
|
dumpErr := dump.Wait()
|
|
// Only an archive read to its end may commit. On any other outcome the stream
|
|
// is closed without COMMIT, which leaves an open transaction for the server to
|
|
// abort when the session ends.
|
|
if bodyErr == nil && dumpErr == nil {
|
|
bodyErr = write(trailer + "COMMIT;\n")
|
|
}
|
|
_ = stdin.Close()
|
|
sqlWait := sql.Wait()
|
|
if sqlWait != nil {
|
|
return fmt.Errorf("postgres: restore: psql: %w: %s", sqlWait, strings.TrimSpace(sqlErr.String()))
|
|
}
|
|
if dumpErr != nil {
|
|
return fmt.Errorf("postgres: restore: pg_restore: %w: %s", dumpErr, strings.TrimSpace(dumpStderr.String()))
|
|
}
|
|
if bodyErr != nil {
|
|
return fmt.Errorf("postgres: restore: %w", bodyErr)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// claimServeExclusive takes the serve lock in its exclusive form, which no
|
|
// serving process can hold at the same time.
|
|
func claimServeExclusive(ctx context.Context, pool *pgxpool.Pool) (func(context.Context), error) {
|
|
conn, err := pool.Acquire(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: serve lock: %w", err)
|
|
}
|
|
var taken bool
|
|
if err := conn.QueryRow(ctx, "SELECT pg_try_advisory_lock(hashtextextended($1, 0))", platformpg.ServeLockKey(ServeLockName)).Scan(&taken); err != nil {
|
|
conn.Release()
|
|
return nil, fmt.Errorf("postgres: serve lock: %w", err)
|
|
}
|
|
if !taken {
|
|
conn.Release()
|
|
return nil, fmt.Errorf("a serving process holds the database")
|
|
}
|
|
return func(ctx context.Context) {
|
|
_, _ = conn.Exec(ctx, "SELECT pg_advisory_unlock(hashtextextended($1, 0))", platformpg.ServeLockKey(ServeLockName))
|
|
conn.Release()
|
|
}, nil
|
|
}
|
|
|
|
// dropStatementsNotIn returns the statements that drop every table the snapshot
|
|
// does not hold, for the restore's transaction to run before the archive.
|
|
func dropStatementsNotIn(ctx context.Context, pool *pgxpool.Pool, keep []string) ([]string, error) {
|
|
rows, err := pool.Query(ctx, `SELECT schemaname, tablename FROM pg_tables
|
|
WHERE schemaname NOT IN ('pg_catalog', 'information_schema')`)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("postgres: restore: list tables: %w", err)
|
|
}
|
|
var drop []pgx.Identifier
|
|
for rows.Next() {
|
|
var schema, table string
|
|
if err := rows.Scan(&schema, &table); err != nil {
|
|
rows.Close()
|
|
return nil, fmt.Errorf("postgres: restore: list tables: %w", err)
|
|
}
|
|
if !slices.Contains(keep, schema+"."+table) {
|
|
drop = append(drop, pgx.Identifier{schema, table})
|
|
}
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("postgres: restore: list tables: %w", err)
|
|
}
|
|
statements := make([]string, 0, len(drop))
|
|
for _, table := range drop {
|
|
statements = append(statements, "DROP TABLE IF EXISTS "+table.Sanitize()+" CASCADE;")
|
|
}
|
|
return statements, nil
|
|
}
|
|
|
|
// snapshotTables extracts schema.table for every TABLE entry of a
|
|
// `pg_restore --list` listing, sorted.
|
|
func snapshotTables(list string) []string {
|
|
var tables []string
|
|
for line := range strings.Lines(list) {
|
|
if strings.HasPrefix(line, ";") {
|
|
continue
|
|
}
|
|
_, entry, ok := strings.Cut(line, ";")
|
|
if !ok {
|
|
continue
|
|
}
|
|
fields := strings.Fields(entry)
|
|
// "tableoid oid TABLE schema name owner"; "TABLE DATA" entries are rows.
|
|
if len(fields) >= 5 && fields[2] == "TABLE" && fields[3] != "DATA" {
|
|
tables = append(tables, fields[3]+"."+fields[4])
|
|
}
|
|
}
|
|
slices.Sort(tables)
|
|
return tables
|
|
}
|
|
|
|
func dumpArgs(dbname, out string) []string {
|
|
return []string{"--format=custom", "--no-password", "--file=" + out, "--dbname=" + dbname}
|
|
}
|
|
|
|
// restoreArgs reads the archive as SQL on stdout, for psql to execute inside one
|
|
// transaction. --clean drops the objects the archive replaces, and the archive
|
|
// carries no transaction control of its own.
|
|
func restoreArgs(from string) []string {
|
|
return []string{"--clean", "--if-exists", "--no-owner", "--no-password", "--file=-", from}
|
|
}
|
|
|
|
// psqlArgs runs a SQL stream with no client configuration and with a failure
|
|
// stopping the session before it reaches the caller's COMMIT.
|
|
func psqlArgs(dbname string) []string {
|
|
return []string{"-X", "-v", "ON_ERROR_STOP=1", "--no-password", "--dbname=" + dbname}
|
|
}
|
|
|
|
// libpqTarget splits a connection URL into the dbname argument and the
|
|
// environment the client tools read, moving the password to PGPASSWORD so it
|
|
// never appears in a process listing. A key=value connection string is passed
|
|
// through as-is.
|
|
func libpqTarget(databaseURL string) (string, []string) {
|
|
u, err := url.Parse(databaseURL)
|
|
if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil {
|
|
return databaseURL, nil
|
|
}
|
|
password, ok := u.User.Password()
|
|
if !ok {
|
|
return databaseURL, nil
|
|
}
|
|
u.User = url.User(u.User.Username())
|
|
return u.String(), []string{"PGPASSWORD=" + password}
|
|
}
|
|
|
|
// runTool runs a PostgreSQL client tool and returns its stdout; a failure
|
|
// carries its stderr, which is where the tools explain themselves.
|
|
func runTool(ctx context.Context, env []string, name string, args ...string) (string, error) {
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
cmd.Env = append(os.Environ(), env...)
|
|
var stdout, stderr bytes.Buffer
|
|
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
|
if err := cmd.Run(); err != nil {
|
|
return "", fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String()))
|
|
}
|
|
return stdout.String(), nil
|
|
}
|